<commit_before>#![feature(test, btree_range, collections_bound)]

// (for test) This is silly...
#![allow(unused_features)]

#[macro_use]
extern crate log;
extern crate crc;
extern crate memmap;
extern crate byteorder;
extern crate env_logger;

#[cfg(test)]
extern crate test;
#[cfg(test)]
extern crate rand;

mod segment;
mod index;
#[cfg(test)]
mod testutil;

use std::collections::{Bound, BTreeMap};
use std::path::{Path, PathBuf};
use std::fs;
use std::io;
use std::mem::swap;
use segment::{Segment, SegmentAppendError};
use index::*;

pub use segment::ReadLimit;
pub use segment::Message;

/// Offset of an appended log segment.
#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)]
pub struct Offset(pub u64);

/// Error enum for commit log Append operation.
#[derive(Debug)]
pub enum AppendError {
    /// The underlying file operations failed during the append attempt.
    Io(io::Error),
    /// A new index was created, but was unable to receive writes
    /// during the append operation. This could point to exhaustion
    /// of machine resources or other I/O issue.
    FreshIndexNotWritable,
    /// A new segment was created, but was unable to receive writes
    /// during the append operation. This could point to exhaustion
    /// of machine resources or other I/O issue.
    FreshSegmentNotWritable,
}

impl From<io::Error> for AppendError {
    fn from(e: io::Error) -> AppendError {
        AppendError::Io(e)
    }
}

/// Starting location of a read
#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)]
pub enum ReadPosition {
    /// Start reading from the initial offset
    Beginning,
    /// Start reading from a specified offset
    Offset(Offset), // TODO: particular pointer
}

/// Error enum for commit log read operation.
#[derive(Debug)]
pub enum ReadError {
    Io(io::Error),
    CorruptLog,
}

impl From<io::Error> for ReadError {
    fn from(e: io::Error) -> ReadError {
        ReadError::Io(e)
    }
}

impl From<segment::MessageError> for ReadError {
    fn from(e: segment::MessageError) -> ReadError {
        match e {
            segment::MessageError::IoError(e) => ReadError::Io(e),
            segment::MessageError::InvalidCRC => ReadError::CorruptLog,
        }
    }
}

/// Commit log options allow customization of the commit
/// log behavior.
#[derive(Clone, Debug)]
pub struct LogOptions {
    log_dir: PathBuf,
    log_max_bytes: usize,
    index_max_bytes: usize,
}

impl LogOptions {
    /// Creates minimal LogOptions with a directory containing the log.
    pub fn new<P>(log_dir: P) -> LogOptions
        where P: AsRef<Path>
    {
        LogOptions {
            log_dir: log_dir.as_ref().to_owned(),
            log_max_bytes: 100 * 1024 * 1024,
            index_max_bytes: 800_000,
        }
    }

    /// Bounds the size of a log segment to a number of bytes.
    #[inline]
    pub fn max_bytes_log(&mut self, bytes: usize) -> &mut LogOptions {
        self.log_max_bytes = bytes;
        self
    }

    /// Bounds the size of an individual memory-mapped index file.
    #[inline]
    pub fn max_index_items(&mut self, items: usize) -> &mut LogOptions {
        self.index_max_bytes = items * INDEX_ENTRY_BYTES;
        self
    }
}

/// The commit log is an append-only data structure that can be used in a variety
/// of use-cases, such as tracking sequences of events, transactions
/// or replicated state machines.
///
/// This implementation of the commit log data structure uses log segments
/// that roll over at pre-defined maximum size boundaries. The messages appended
/// to the log have a unique, monotonically increasing offset that can be used as
/// a pointer to a log entry.
///
/// The index of the commit log logically stores the offset to a position in the
/// log segment corresponding. The index and segments are separated, in that a
/// segment file does not necessarily correspond to one particular segment file,
/// it could contain file pointers to many segment files. In addition, index files
/// are memory-mapped for efficient read and write access.
pub struct CommitLog {
    closed_segments: BTreeMap<u64, Segment>,
    closed_indexes: BTreeMap<u64, Index>,

    active_segment: Segment,
    active_index: Index,
    options: LogOptions,
}

impl CommitLog {
    pub fn new(opts: LogOptions) -> io::Result<CommitLog> {
        // TODO: figure out what's already been written to
        fs::create_dir_all(&opts.log_dir).unwrap_or(());

        info!("Opening log at path {:?}", &opts.log_dir.to_str());
        let seg = Segment::new(&opts.log_dir, 0u64, opts.log_max_bytes)?;
        let ind = Index::new(&opts.log_dir, 0u64, opts.index_max_bytes)?;

        Ok(CommitLog {
            closed_segments: BTreeMap::new(),
            closed_indexes: BTreeMap::new(),

            active_segment: seg,
            active_index: ind,
            options: opts,
        })
    }

    /// Appends a log entry to the commit log, returning the offset of the appended entry.
    pub fn append(&mut self, payload: &[u8]) -> Result<Offset, AppendError> {
        // first write to the current segment
        let meta = try!(self.active_segment
            .append(payload)
            .or_else(|e| {
                match e {
                    // if the log is full, gracefully close the current segment
                    // and create new one starting from the new offset
                    SegmentAppendError::LogFull => {
                        self.active_segment.flush_sync()?;
                        let next_offset = self.active_segment.next_offset();

                        info!("Starting new segment at offset {}", next_offset);

                        let mut seg = Segment::new(&self.options.log_dir,
                                                   next_offset,
                                                   self.options.log_max_bytes)?;

                        // set the active segment to the new segment,
                        // swap in order to insert the segment into
                        // the closed segments tree
                        swap(&mut seg, &mut self.active_segment);
                        self.closed_segments.insert(seg.starting_offset(), seg);

                        // try again, giving up if we have to
                        self.active_segment
                            .append(payload)
                            .map_err(|_| AppendError::FreshSegmentNotWritable)
                    }
                    SegmentAppendError::IoError(e) => Err(AppendError::Io(e)),
                }
            }));

        // write to the index
        self.active_index
            .append(meta.offset(), meta.file_pos())
            .or_else(|e| {
                match e {
                    // if the index is full, close the current index and open a new index
                    IndexWriteError::IndexFull => {
                        info!("Starting new index at offset {}", meta.offset());

                        try!(self.active_index.set_readonly());
                        let mut ind = try!(Index::new(&self.options.log_dir,
                                                      meta.offset(),
                                                      self.options.index_max_bytes));

                        // set the active index to the new index,
                        // swap in order to insert the index into
                        // the closed index tree
                        swap(&mut ind, &mut self.active_index);
                        self.closed_indexes.insert(ind.starting_offset(), ind);

                        // if the new index cannot append, we're out of luck
                        self.active_index
                            .append(meta.offset(), meta.file_pos())
                            .map_err(|_| AppendError::FreshIndexNotWritable)
                    }
                    IndexWriteError::OffsetLessThanBase => unreachable!(),
                }
            })?;

        Ok(Offset(meta.offset()))
    }

    pub fn read(&mut self,
                start: ReadPosition,
                limit: ReadLimit)
                -> Result<Vec<Message>, ReadError> {
        let start_off = match start {
            ReadPosition::Beginning => 0,
            ReadPosition::Offset(Offset(v)) => v,
        };

        // find the appropriate index
        // TODO: change index find to be >= to offset
        let active_start_off = self.active_index.starting_offset();

        // find the file position from the index
        let index_entry_res = if start_off >= active_start_off {
            info!("using active index");
            self.active_index.find(start_off)
        } else {
            info!("using old index");
            let found_index = self.closed_indexes
                .range(Bound::Unbounded, Bound::Included(&start_off))
                .next_back();
            found_index.and_then(|(_, i)| i.find(start_off))
        };

        let file_pos = match index_entry_res {
            Some(e) => e.file_position(),
            None => {
                info!("No index entry found for {}", start_off);
                return Ok(Vec::with_capacity(0));
            }
        };

        // find the correct segment
        let active_seg_start_off = self.active_segment.starting_offset();
        if start_off >= active_seg_start_off {
            info!("Reading from active index at file pos {}", file_pos);
            Ok(self.active_segment.read(file_pos, limit)?)
        } else {
            let mut r = self.closed_segments
                .range_mut(Bound::Unbounded, Bound::Included(&start_off));
            match r.next_back() {
                Some((_, ref mut s)) => {
                    info!("Reading from old index at file pos {}", file_pos);
                    Ok(s.read(file_pos, limit)?)
                }
                _ => {
                    info!("No index found for off {}", start_off);
                    Ok(Vec::with_capacity(0))
                }
            }
        }
    }

    pub fn flush(&mut self) -> io::Result<()> {
        self.active_segment.flush_sync()?;
        self.active_index.flush_sync()
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use super::testutil::*;
    use std::fs;
    use std::collections::HashSet;
    use env_logger;

    #[test]
    pub fn append() {
        let dir = TestDir::new();
        let mut log = CommitLog::new(LogOptions::new(&dir)).unwrap();
        assert_eq!(log.append(b"123456").unwrap(), Offset(0));
        assert_eq!(log.append(b"abcdefg").unwrap(), Offset(1));
        assert_eq!(log.append(b"foobarbaz").unwrap(), Offset(2));
        assert_eq!(log.append(b"bing").unwrap(), Offset(3));
        log.flush().unwrap();
    }

    #[test]
    pub fn append_new_segment() {
        let dir = TestDir::new();
        let mut opts = LogOptions::new(&dir);
        opts.max_bytes_log(52);

        {
            let mut log = CommitLog::new(opts).unwrap();
            // first 2 entries fit (both 26 bytes with encoding)
            log.append(b"0123456789").unwrap();
            log.append(b"0123456789").unwrap();

            // this one should roll the log
            log.append(b"0123456789").unwrap();
            log.flush().unwrap();
        }

        let files = fs::read_dir(&dir)
            .unwrap()
            .map(|e| e.unwrap().path().file_name().unwrap().to_str().unwrap().to_string())
            .collect::<HashSet<String>>();

        let expected =
            ["00000000000000000000.index", "00000000000000000000.log", "00000000000000000002.log"]
                .iter()
                .cloned()
                .map(|s| s.to_string())
                .collect::<HashSet<String>>();

        assert_eq!(files.intersection(&expected).count(), 3);
    }

    #[test]
    pub fn append_new_index() {
        let dir = TestDir::new();
        let mut opts = LogOptions::new(&dir);
        opts.max_index_items(2);

        {
            let mut log = CommitLog::new(opts).unwrap();
            // first 2 entries fit
            log.append(b"0123456789").unwrap();
            log.append(b"0123456789").unwrap();

            // this one should roll the index, but not the segment
            log.append(b"0123456789").unwrap();
            log.flush().unwrap();
        }

        let files = fs::read_dir(&dir)
            .unwrap()
            .map(|e| e.unwrap().path().file_name().unwrap().to_str().unwrap().to_string())
            .collect::<HashSet<String>>();

        let expected = ["00000000000000000000.index",
                        "00000000000000000000.log",
                        "00000000000000000002.index"]
            .iter()
            .cloned()
            .map(|s| s.to_string())
            .collect::<HashSet<String>>();

        assert_eq!(files.intersection(&expected).count(), 3);
    }

    #[test]
    pub fn read_entries() {
        env_logger::init().unwrap();

        let dir = TestDir::new();
        let mut opts = LogOptions::new(&dir);
        opts.max_index_items(20);
        opts.max_bytes_log(1000);
        let mut log = CommitLog::new(opts).unwrap();

        for i in 0..100 {
            let s = format!("some data {}", i);
            log.append(s.as_bytes()).unwrap();
        }
        log.flush().unwrap();

        {
            let active_index_read =
                log.read(ReadPosition::Offset(Offset(82)), ReadLimit::Messages(5)).unwrap();
            assert_eq!(5, active_index_read.len());
            assert_eq!(vec![82, 83, 84, 85, 86],
                       active_index_read.into_iter().map(|v| v.offset()).collect::<Vec<_>>());
        }

        {
            let old_index_read = log.read(ReadPosition::Offset(Offset(5)), ReadLimit::Messages(5))
                .unwrap();
            assert_eq!(5, old_index_read.len());
            assert_eq!(vec![5, 6, 7, 8, 9],
                       old_index_read.into_iter().map(|v| v.offset()).collect::<Vec<_>>());
        }

        // read at the boundary (not going to get full message limit)
        {
            // log rolls at offset 36
            let boundary_read = log.read(ReadPosition::Offset(Offset(33)), ReadLimit::Messages(5))
                .unwrap();
            assert_eq!(3, boundary_read.len());
            assert_eq!(vec![33, 34, 35],
                       boundary_read.into_iter().map(|v| v.offset()).collect::<Vec<_>>());
        }

    }
}
<commit_msg>Changing log levels<commit_after>#![feature(test, btree_range, collections_bound)]

// (for test) This is silly...
#![allow(unused_features)]

#[macro_use]
extern crate log;
extern crate crc;
extern crate memmap;
extern crate byteorder;
extern crate env_logger;

#[cfg(test)]
extern crate test;
#[cfg(test)]
extern crate rand;

mod segment;
mod index;
#[cfg(test)]
mod testutil;

use std::collections::{Bound, BTreeMap};
use std::path::{Path, PathBuf};
use std::fs;
use std::io;
use std::mem::swap;
use segment::{Segment, SegmentAppendError};
use index::*;

pub use segment::ReadLimit;
pub use segment::Message;

/// Offset of an appended log segment.
#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)]
pub struct Offset(pub u64);

/// Error enum for commit log Append operation.
#[derive(Debug)]
pub enum AppendError {
    /// The underlying file operations failed during the append attempt.
    Io(io::Error),
    /// A new index was created, but was unable to receive writes
    /// during the append operation. This could point to exhaustion
    /// of machine resources or other I/O issue.
    FreshIndexNotWritable,
    /// A new segment was created, but was unable to receive writes
    /// during the append operation. This could point to exhaustion
    /// of machine resources or other I/O issue.
    FreshSegmentNotWritable,
}

impl From<io::Error> for AppendError {
    fn from(e: io::Error) -> AppendError {
        AppendError::Io(e)
    }
}

/// Starting location of a read
#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)]
pub enum ReadPosition {
    /// Start reading from the initial offset
    Beginning,
    /// Start reading from a specified offset
    Offset(Offset), // TODO: particular pointer
}

/// Error enum for commit log read operation.
#[derive(Debug)]
pub enum ReadError {
    Io(io::Error),
    CorruptLog,
}

impl From<io::Error> for ReadError {
    fn from(e: io::Error) -> ReadError {
        ReadError::Io(e)
    }
}

impl From<segment::MessageError> for ReadError {
    fn from(e: segment::MessageError) -> ReadError {
        match e {
            segment::MessageError::IoError(e) => ReadError::Io(e),
            segment::MessageError::InvalidCRC => ReadError::CorruptLog,
        }
    }
}

/// Commit log options allow customization of the commit
/// log behavior.
#[derive(Clone, Debug)]
pub struct LogOptions {
    log_dir: PathBuf,
    log_max_bytes: usize,
    index_max_bytes: usize,
}

impl LogOptions {
    /// Creates minimal LogOptions with a directory containing the log.
    pub fn new<P>(log_dir: P) -> LogOptions
        where P: AsRef<Path>
    {
        LogOptions {
            log_dir: log_dir.as_ref().to_owned(),
            log_max_bytes: 100 * 1024 * 1024,
            index_max_bytes: 800_000,
        }
    }

    /// Bounds the size of a log segment to a number of bytes.
    #[inline]
    pub fn max_bytes_log(&mut self, bytes: usize) -> &mut LogOptions {
        self.log_max_bytes = bytes;
        self
    }

    /// Bounds the size of an individual memory-mapped index file.
    #[inline]
    pub fn max_index_items(&mut self, items: usize) -> &mut LogOptions {
        self.index_max_bytes = items * INDEX_ENTRY_BYTES;
        self
    }
}

/// The commit log is an append-only data structure that can be used in a variety
/// of use-cases, such as tracking sequences of events, transactions
/// or replicated state machines.
///
/// This implementation of the commit log data structure uses log segments
/// that roll over at pre-defined maximum size boundaries. The messages appended
/// to the log have a unique, monotonically increasing offset that can be used as
/// a pointer to a log entry.
///
/// The index of the commit log logically stores the offset to a position in the
/// log segment corresponding. The index and segments are separated, in that a
/// segment file does not necessarily correspond to one particular segment file,
/// it could contain file pointers to many segment files. In addition, index files
/// are memory-mapped for efficient read and write access.
pub struct CommitLog {
    closed_segments: BTreeMap<u64, Segment>,
    closed_indexes: BTreeMap<u64, Index>,

    active_segment: Segment,
    active_index: Index,
    options: LogOptions,
}

impl CommitLog {
    pub fn new(opts: LogOptions) -> io::Result<CommitLog> {
        // TODO: figure out what's already been written to
        fs::create_dir_all(&opts.log_dir).unwrap_or(());

        info!("Opening log at path {:?}", &opts.log_dir.to_str());
        let seg = Segment::new(&opts.log_dir, 0u64, opts.log_max_bytes)?;
        let ind = Index::new(&opts.log_dir, 0u64, opts.index_max_bytes)?;

        Ok(CommitLog {
            closed_segments: BTreeMap::new(),
            closed_indexes: BTreeMap::new(),

            active_segment: seg,
            active_index: ind,
            options: opts,
        })
    }

    /// Appends a log entry to the commit log, returning the offset of the appended entry.
    pub fn append(&mut self, payload: &[u8]) -> Result<Offset, AppendError> {
        // first write to the current segment
        let meta = self.active_segment
            .append(payload)
            .or_else(|e| {
                match e {
                    // if the log is full, gracefully close the current segment
                    // and create new one starting from the new offset
                    SegmentAppendError::LogFull => {
                        self.active_segment.flush_sync()?;
                        let next_offset = self.active_segment.next_offset();

                        info!("Starting new segment at offset {}", next_offset);

                        let mut seg = Segment::new(&self.options.log_dir,
                                                   next_offset,
                                                   self.options.log_max_bytes)?;

                        // set the active segment to the new segment,
                        // swap in order to insert the segment into
                        // the closed segments tree
                        swap(&mut seg, &mut self.active_segment);
                        self.closed_segments.insert(seg.starting_offset(), seg);

                        // try again, giving up if we have to
                        self.active_segment
                            .append(payload)
                            .map_err(|_| AppendError::FreshSegmentNotWritable)
                    }
                    SegmentAppendError::IoError(e) => Err(AppendError::Io(e)),
                }
            })?;

        // write to the index
        self.active_index
            .append(meta.offset(), meta.file_pos())
            .or_else(|e| {
                match e {
                    // if the index is full, close the current index and open a new index
                    IndexWriteError::IndexFull => {
                        info!("Starting new index at offset {}", meta.offset());

                        try!(self.active_index.set_readonly());
                        let mut ind = try!(Index::new(&self.options.log_dir,
                                                      meta.offset(),
                                                      self.options.index_max_bytes));

                        // set the active index to the new index,
                        // swap in order to insert the index into
                        // the closed index tree
                        swap(&mut ind, &mut self.active_index);
                        self.closed_indexes.insert(ind.starting_offset(), ind);

                        // if the new index cannot append, we're out of luck
                        self.active_index
                            .append(meta.offset(), meta.file_pos())
                            .map_err(|_| AppendError::FreshIndexNotWritable)
                    }
                    IndexWriteError::OffsetLessThanBase => unreachable!(),
                }
            })?;

        Ok(Offset(meta.offset()))
    }

    pub fn read(&mut self,
                start: ReadPosition,
                limit: ReadLimit)
                -> Result<Vec<Message>, ReadError> {
        let start_off = match start {
            ReadPosition::Beginning => 0,
            ReadPosition::Offset(Offset(v)) => v,
        };

        // TODO: change index find to be >= to offset

        // find the file position from the index
        let active_start_off = self.active_index.starting_offset();
        let index_entry_res = if start_off >= active_start_off {
            trace!("Reading offset {} from active index", start_off);
            self.active_index.find(start_off)
        } else {
            trace!("Reading offset {} from old index", start_off);
            let found_index = self.closed_indexes
                .range(Bound::Unbounded, Bound::Included(&start_off))
                .next_back();
            found_index.and_then(|(_, i)| i.find(start_off))
        };

        let file_pos = match index_entry_res {
            Some(e) => e.file_position(),
            None => {
                info!("No index entry found for {}", start_off);
                return Ok(Vec::with_capacity(0));
            }
        };

        // find the correct segment and read the log entry
        let active_seg_start_off = self.active_segment.starting_offset();

        if start_off >= active_seg_start_off {
            trace!("Reading from active index at file pos {}", file_pos);
            Ok(self.active_segment.read(file_pos, limit)?)
        } else {
            let mut r = self.closed_segments
                .range_mut(Bound::Unbounded, Bound::Included(&start_off));
            match r.next_back() {
                Some((_, ref mut s)) => {
                    trace!("Reading messages from old index at file pos {}", file_pos);
                    Ok(s.read(file_pos, limit)?)
                }
                _ => {
                    warn!("No segment found for offset {}", start_off);
                    Ok(Vec::with_capacity(0))
                }
            }
        }
    }

    pub fn flush(&mut self) -> io::Result<()> {
        self.active_segment.flush_sync()?;
        self.active_index.flush_sync()
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use super::testutil::*;
    use std::fs;
    use std::collections::HashSet;
    use env_logger;

    #[test]
    pub fn append() {
        let dir = TestDir::new();
        let mut log = CommitLog::new(LogOptions::new(&dir)).unwrap();
        assert_eq!(log.append(b"123456").unwrap(), Offset(0));
        assert_eq!(log.append(b"abcdefg").unwrap(), Offset(1));
        assert_eq!(log.append(b"foobarbaz").unwrap(), Offset(2));
        assert_eq!(log.append(b"bing").unwrap(), Offset(3));
        log.flush().unwrap();
    }

    #[test]
    pub fn append_new_segment() {
        let dir = TestDir::new();
        let mut opts = LogOptions::new(&dir);
        opts.max_bytes_log(52);

        {
            let mut log = CommitLog::new(opts).unwrap();
            // first 2 entries fit (both 26 bytes with encoding)
            log.append(b"0123456789").unwrap();
            log.append(b"0123456789").unwrap();

            // this one should roll the log
            log.append(b"0123456789").unwrap();
            log.flush().unwrap();
        }

        let files = fs::read_dir(&dir)
            .unwrap()
            .map(|e| e.unwrap().path().file_name().unwrap().to_str().unwrap().to_string())
            .collect::<HashSet<String>>();

        let expected =
            ["00000000000000000000.index", "00000000000000000000.log", "00000000000000000002.log"]
                .iter()
                .cloned()
                .map(|s| s.to_string())
                .collect::<HashSet<String>>();

        assert_eq!(files.intersection(&expected).count(), 3);
    }

    #[test]
    pub fn append_new_index() {
        let dir = TestDir::new();
        let mut opts = LogOptions::new(&dir);
        opts.max_index_items(2);

        {
            let mut log = CommitLog::new(opts).unwrap();
            // first 2 entries fit
            log.append(b"0123456789").unwrap();
            log.append(b"0123456789").unwrap();

            // this one should roll the index, but not the segment
            log.append(b"0123456789").unwrap();
            log.flush().unwrap();
        }

        let files = fs::read_dir(&dir)
            .unwrap()
            .map(|e| e.unwrap().path().file_name().unwrap().to_str().unwrap().to_string())
            .collect::<HashSet<String>>();

        let expected = ["00000000000000000000.index",
                        "00000000000000000000.log",
                        "00000000000000000002.index"]
            .iter()
            .cloned()
            .map(|s| s.to_string())
            .collect::<HashSet<String>>();

        assert_eq!(files.intersection(&expected).count(), 3);
    }

    #[test]
    pub fn read_entries() {
        env_logger::init().unwrap();

        let dir = TestDir::new();
        let mut opts = LogOptions::new(&dir);
        opts.max_index_items(20);
        opts.max_bytes_log(1000);
        let mut log = CommitLog::new(opts).unwrap();

        for i in 0..100 {
            let s = format!("some data {}", i);
            log.append(s.as_bytes()).unwrap();
        }
        log.flush().unwrap();

        {
            let active_index_read =
                log.read(ReadPosition::Offset(Offset(82)), ReadLimit::Messages(5)).unwrap();
            assert_eq!(5, active_index_read.len());
            assert_eq!(vec![82, 83, 84, 85, 86],
                       active_index_read.into_iter().map(|v| v.offset()).collect::<Vec<_>>());
        }

        {
            let old_index_read = log.read(ReadPosition::Offset(Offset(5)), ReadLimit::Messages(5))
                .unwrap();
            assert_eq!(5, old_index_read.len());
            assert_eq!(vec![5, 6, 7, 8, 9],
                       old_index_read.into_iter().map(|v| v.offset()).collect::<Vec<_>>());
        }

        // read at the boundary (not going to get full message limit)
        {
            // log rolls at offset 36
            let boundary_read = log.read(ReadPosition::Offset(Offset(33)), ReadLimit::Messages(5))
                .unwrap();
            assert_eq!(3, boundary_read.len());
            assert_eq!(vec![33, 34, 35],
                       boundary_read.into_iter().map(|v| v.offset()).collect::<Vec<_>>());
        }

    }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate nom;

use std::str;
use std::str::FromStr;
use std::fmt;
use std::fs::File;
use std::io::prelude::*;

#[derive(Debug, Clone)]
pub struct Sexp {
    element:Element,
    meta:Meta,
}

#[derive(Debug, Clone)]
pub enum Element {
    String(String),
    List(Vec<Sexp>),
    Empty,
}

#[derive(Debug, Clone)]
pub struct Meta {
    indent:String,
    nl:usize,
}

pub type ERes<T> = Result<T, String>;

impl Sexp {

    fn new_empty() -> Sexp {
        Sexp { element:Element::Empty, meta:Meta { indent:String::from(""), nl:0,} }
    }

    fn new(element:Element, indent:String, nl:usize) -> Sexp {
        Sexp { element:element, meta:Meta { indent:indent, nl:nl, } }
    }

    pub fn list(&self) -> ERes<&Vec<Sexp> > {
        match self.element {
            Element::List(ref v) => Ok(v),
            _ => Err(format!("not a list: {}", self))
        }
    }

    pub fn string(&self) -> ERes<&String> {
        match self.element {
            Element::String(ref s) => Ok(s),
            _ => Err(format!("not a string: {}", self))
        }
    }

    pub fn f(&self) -> ERes<f64> {
        let s = try!(self.string());
        match f64::from_str(&s) {
            Ok(f) => Ok(f),
            _ => Err(format!("Error parsing float"))
        }
    }

    pub fn i(&self) -> ERes<i64> {
        let s = try!(self.string());
        match i64::from_str(&s) {
            Ok(f) => Ok(f),
            _ => Err(format!("Error parsing int"))
        }
    }

    pub fn list_name(&self) -> ERes<&String> {
        let l = try!(self.list());
        let l = &l[..];
        let a = try!(l[0].string());
        Ok(a)
    }

    pub fn slice_atom(&self, s:&str) -> ERes<&[Sexp]> {
        let v = try!(self.list());
        let v2 =&v[..];
        let st = try!(v2[0].string());
        if st != s {
            return Err(format!("list doesn't start with {}, but with {}", s, st))
        };
        Ok(&v[1..])
    }
}

impl fmt::Display for Sexp {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        try!(write!(f, "{}", self.meta.indent));
        try!(match self.element {
            Element::String(ref s) => {
                if s.contains("(") || s.contains(" ") {
                    write!(f,"\"{}\"", s)
                } else {
                    write!(f,"{}", s)
                }
            },
            Element::List(ref v) => {
                try!(write!(f, "("));
                let mut prev_nl = 0;
                for (i, x) in v.iter().enumerate() {
                    let s = if i == 0 || x.meta.indent.len() > 0 || prev_nl > 0 { "" } else { " " };
                    try!(write!(f, "{}{}", s, x));
                    prev_nl = x.meta.nl;
                }
                write!(f, ")")
            },
            Element::Empty => Ok(())
        });
        for _ in 0..self.meta.nl {
            try!(writeln!(f,""));
        }
        Ok(())
    }
}

pub fn display_string(s:&String) -> String {
    if s.contains("(") || s.contains(" ") || s.len() == 0 {
        format!("\"{}\"", s)
    } else {
        s.clone()
    }
}

pub fn parse_str(sexp: &str) -> Result<Sexp, String> {
    if sexp.len() == 0 {
        return Ok(Sexp::new_empty())
    }
    match parse_sexp(&sexp.as_bytes()[..]) {
        nom::IResult::Done(_, c) => Ok(c),
        nom::IResult::Error(err) => {
            match err {
                nom::Err::Position(kind,p) =>
                    Err(format!("parse error: {:?} |{}|", kind, str::from_utf8(p).unwrap())),
                _ => Err(format!("parse error"))
            }
        },
        nom::IResult::Incomplete(x) => Err(format!("incomplete: {:?}", x)),
    }
}

fn read_file(name: &str) -> Result<String, std::io::Error> {
    let mut f = try!(File::open(name));
    let mut s = String::new();
    try!(f.read_to_string(&mut s));
    Ok(s)
}

pub fn parse_file(name: &str) -> ERes<Sexp> {
    let s = try!(match read_file(name) {
        Ok(s) => Ok(s),
        Err(x) => Err(format!("{:?}", x))
    });
    parse_str(&s[..])
}

named!(parse_qstring<String>,
       map_res!(
           map_res!(
               delimited!(char!('\"'), is_not!("\""), char!('\"')),
               str::from_utf8),
           FromStr::from_str)
       );

named!(parse_bare_string<String>,
       map_res!(
           map_res!(
               is_not!(b")( \r\n"),
               str::from_utf8),
           FromStr::from_str)
       );

named!(parse_string<String>,
       alt!(parse_qstring | parse_bare_string)
       );

named!(parse_list<Vec<Sexp> >,
       chain!(
           char!('(') ~
           v: many0!(parse_sexp) ~
           char!(')'),
           || v)
       );

named!(line_ending<usize>,
       chain!(
           opt!(nom::space) ~
               c: opt!(is_a!(b"\r\n"))
               , || match c { None => 0, Some(ref x) => x.len(), }
               )
       );

named!(parse_sexp<Sexp>,
           chain!(
               indent: opt!(nom::space) ~
                   s: alt!(map!(parse_list,Element::List) | map!(parse_string,Element::String)) ~
                   nl: line_ending
                   ,
               || {
                   let indent = match indent {
                       None => String::new(),
                       Some(x) => String::from(str::from_utf8(x).unwrap()),
                   };
                   Sexp::new(s, indent, nl)
               })
       );

// internal tests
#[test]
fn test_qstring1() {
    assert_eq!(parse_string(&b"\"hello world\""[..]), nom::IResult::Done(&b""[..], String::from("hello world")));
}

#[test]
#[should_panic(expected="assertion failed: `(left == right)` (left: `Incomplete(Size(1))`, right: `Done([], \"hello\")`)")]
fn test_qstring2() {
    assert_eq!(parse_string(&b"\"hello"[..]), nom::IResult::Done(&b""[..], String::from("hello")));
}

#[test]
fn test_string1() {
    assert_eq!(parse_string(&b"hello world"[..]), nom::IResult::Done(&b" world"[..], String::from("hello")));
}

#[cfg(test)]
mod tests {
    use super::*;

    #[allow(dead_code)]
    fn check_parse_res(s: &str, o:&str) {
        let e = parse_str(s).unwrap();
        let t = format!("{}", e);
        assert_eq!(o, t)
    }
    #[allow(dead_code)]
    fn check_parse(s: &str) {
        let e = parse_str(s).unwrap();
        let t = format!("{}", e);
        assert_eq!(s, t)
    }

    #[allow(dead_code)]
    fn parse_fail(s: &str) {
        parse_str(s).unwrap();
    }

    #[test]
    fn test_empty() { check_parse("") }

    #[test]
    fn test_empty_qstring() { check_parse("(hello \"\")") }

    #[test]
    fn test_minimal() { check_parse("()") }

    #[test]
    fn test_string() { check_parse("hello") }

    #[test]
    fn test_qstring_a() { check_parse_res("\"hello\"", "hello") }

    #[test]
    fn test_qstring_a2() { check_parse("\"hello world\"") }

    #[test]
    fn test_qstring_a3() { check_parse("\"hello(world)\"") }

    #[test]
    fn test_number() { check_parse("1.3") }

    #[test]
    fn test_float_vs_int() { check_parse("2.0") }

    #[test]
    fn test_double() { check_parse("(())") }

    #[test]
    fn test_br_string() { check_parse("(world)") }

    #[test]
    fn test_br_qstring() { check_parse_res("(\"world\")", "(world)") }

    #[test]
    fn test_br_int() { check_parse("(42)") }

    #[test]
    fn test_br_float() { check_parse("(12.7)") }

    #[test]
    fn test_br_qbrstring() { check_parse("(\"(()\")") }

    #[test]
    fn test_number_string() { check_parse("567A_WZ") }

    #[test]
    #[should_panic(expected="called `Result::unwrap()` on an `Err` value: \"incomplete: Size(2)\"")]
    fn test_invalid1() { parse_fail("(") }

    #[test]
    #[should_panic(expected="called `Result::unwrap()` on an `Err` value: \"parse error: Alt |)|\"")]
    fn test_invalid2() { parse_fail(")") }

    #[test]
    #[should_panic(expected="called `Result::unwrap()` on an `Err` value: \"incomplete: Size(1)\"")]
    fn test_invalid3() { parse_fail("\"hello") }

    #[test]
    fn test_complex() { check_parse("(module SWITCH_3W_SIDE_MMP221-R (layer F.Cu) (descr \"\") (pad 1 thru_hole rect (size 1.2 1.2) (at -2.5 -1.6 0) (layers *.Cu *.Mask) (drill 0.8)) (pad 2 thru_hole rect (size 1.2 1.2) (at 0.0 -1.6 0) (layers *.Cu *.Mask) (drill 0.8)) (pad 3 thru_hole rect (size 1.2 1.2) (at 2.5 -1.6 0) (layers *.Cu *.Mask) (drill 0.8)) (pad 5 thru_hole rect (size 1.2 1.2) (at 0.0 1.6 0) (layers *.Cu *.Mask) (drill 0.8)) (pad 6 thru_hole rect (size 1.2 1.2) (at -2.5 1.6 0) (layers *.Cu *.Mask) (drill 0.8)) (pad 4 thru_hole rect (size 1.2 1.2) (at 2.5 1.6 0) (layers *.Cu *.Mask) (drill 0.8)) (fp_line (start -4.5 -1.75) (end 4.5 -1.75) (layer F.SilkS) (width 0.127)) (fp_line (start 4.5 -1.75) (end 4.5 1.75) (layer F.SilkS) (width 0.127)) (fp_line (start 4.5 1.75) (end -4.5 1.75) (layer F.SilkS) (width 0.127)) (fp_line (start -4.5 1.75) (end -4.5 -1.75) (layer F.SilkS) (width 0.127)))") }

    #[test]
    fn test_multiline() {
        check_parse("\
(hello \"test it\"
    (foo bar)
    (mars venus)
)
")
    }

    #[test]
    fn test_multiline2() {
        check_parse("\
(hello world
  mad
    (world)
  not)")
    }

    #[test]
    fn test_multiline_two_empty() {
        check_parse("\
(hello

world)")
    }
}

<commit_msg>remark about just space on empty line<commit_after>#[macro_use]
extern crate nom;

use std::str;
use std::str::FromStr;
use std::fmt;
use std::fs::File;
use std::io::prelude::*;

#[derive(Debug, Clone)]
pub struct Sexp {
    element:Element,
    meta:Meta,
}

#[derive(Debug, Clone)]
pub enum Element {
    String(String),
    List(Vec<Sexp>),
    Empty,
}

#[derive(Debug, Clone)]
pub struct Meta {
    indent:String,
    nl:usize,
}

pub type ERes<T> = Result<T, String>;

impl Sexp {

    fn new_empty() -> Sexp {
        Sexp { element:Element::Empty, meta:Meta { indent:String::from(""), nl:0,} }
    }

    fn new(element:Element, indent:String, nl:usize) -> Sexp {
        Sexp { element:element, meta:Meta { indent:indent, nl:nl, } }
    }

    pub fn list(&self) -> ERes<&Vec<Sexp> > {
        match self.element {
            Element::List(ref v) => Ok(v),
            _ => Err(format!("not a list: {}", self))
        }
    }

    pub fn string(&self) -> ERes<&String> {
        match self.element {
            Element::String(ref s) => Ok(s),
            _ => Err(format!("not a string: {}", self))
        }
    }

    pub fn f(&self) -> ERes<f64> {
        let s = try!(self.string());
        match f64::from_str(&s) {
            Ok(f) => Ok(f),
            _ => Err(format!("Error parsing float"))
        }
    }

    pub fn i(&self) -> ERes<i64> {
        let s = try!(self.string());
        match i64::from_str(&s) {
            Ok(f) => Ok(f),
            _ => Err(format!("Error parsing int"))
        }
    }

    pub fn list_name(&self) -> ERes<&String> {
        let l = try!(self.list());
        let l = &l[..];
        let a = try!(l[0].string());
        Ok(a)
    }

    pub fn slice_atom(&self, s:&str) -> ERes<&[Sexp]> {
        let v = try!(self.list());
        let v2 =&v[..];
        let st = try!(v2[0].string());
        if st != s {
            return Err(format!("list doesn't start with {}, but with {}", s, st))
        };
        Ok(&v[1..])
    }
}

impl fmt::Display for Sexp {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        try!(write!(f, "{}", self.meta.indent));
        try!(match self.element {
            Element::String(ref s) => {
                if s.contains("(") || s.contains(" ") {
                    write!(f,"\"{}\"", s)
                } else {
                    write!(f,"{}", s)
                }
            },
            Element::List(ref v) => {
                try!(write!(f, "("));
                let mut prev_nl = 0;
                for (i, x) in v.iter().enumerate() {
                    let s = if i == 0 || x.meta.indent.len() > 0 || prev_nl > 0 { "" } else { " " };
                    try!(write!(f, "{}{}", s, x));
                    prev_nl = x.meta.nl;
                }
                write!(f, ")")
            },
            Element::Empty => Ok(())
        });
        for _ in 0..self.meta.nl {
            try!(writeln!(f,""));
        }
        Ok(())
    }
}

pub fn display_string(s:&String) -> String {
    if s.contains("(") || s.contains(" ") || s.len() == 0 {
        format!("\"{}\"", s)
    } else {
        s.clone()
    }
}

pub fn parse_str(sexp: &str) -> Result<Sexp, String> {
    if sexp.len() == 0 {
        return Ok(Sexp::new_empty())
    }
    match parse_sexp(&sexp.as_bytes()[..]) {
        nom::IResult::Done(_, c) => Ok(c),
        nom::IResult::Error(err) => {
            match err {
                nom::Err::Position(kind,p) =>
                    Err(format!("parse error: {:?} |{}|", kind, str::from_utf8(p).unwrap())),
                _ => Err(format!("parse error"))
            }
        },
        nom::IResult::Incomplete(x) => Err(format!("incomplete: {:?}", x)),
    }
}

fn read_file(name: &str) -> Result<String, std::io::Error> {
    let mut f = try!(File::open(name));
    let mut s = String::new();
    try!(f.read_to_string(&mut s));
    Ok(s)
}

pub fn parse_file(name: &str) -> ERes<Sexp> {
    let s = try!(match read_file(name) {
        Ok(s) => Ok(s),
        Err(x) => Err(format!("{:?}", x))
    });
    parse_str(&s[..])
}

named!(parse_qstring<String>,
       map_res!(
           map_res!(
               delimited!(char!('\"'), is_not!("\""), char!('\"')),
               str::from_utf8),
           FromStr::from_str)
       );

named!(parse_bare_string<String>,
       map_res!(
           map_res!(
               is_not!(b")( \r\n"),
               str::from_utf8),
           FromStr::from_str)
       );

named!(parse_string<String>,
       alt!(parse_qstring | parse_bare_string)
       );

named!(parse_list<Vec<Sexp> >,
       chain!(
           char!('(') ~
           v: many0!(parse_sexp) ~
           char!(')'),
           || v)
       );

// TODO: consider lines with just spaces and a nl as also nl
named!(line_ending<usize>,
       chain!(
           opt!(nom::space) ~
               c: opt!(is_a!(b"\r\n"))
               , || match c { None => 0, Some(ref x) => x.len(), }
               )
       );

named!(parse_sexp<Sexp>,
           chain!(
               indent: opt!(nom::space) ~
                   s: alt!(map!(parse_list,Element::List) | map!(parse_string,Element::String)) ~
                   nl: line_ending
                   ,
               || {
                   let indent = match indent {
                       None => String::new(),
                       Some(x) => String::from(str::from_utf8(x).unwrap()),
                   };
                   Sexp::new(s, indent, nl)
               })
       );

// internal tests
#[test]
fn test_qstring1() {
    assert_eq!(parse_string(&b"\"hello world\""[..]), nom::IResult::Done(&b""[..], String::from("hello world")));
}

#[test]
#[should_panic(expected="assertion failed: `(left == right)` (left: `Incomplete(Size(1))`, right: `Done([], \"hello\")`)")]
fn test_qstring2() {
    assert_eq!(parse_string(&b"\"hello"[..]), nom::IResult::Done(&b""[..], String::from("hello")));
}

#[test]
fn test_string1() {
    assert_eq!(parse_string(&b"hello world"[..]), nom::IResult::Done(&b" world"[..], String::from("hello")));
}

#[cfg(test)]
mod tests {
    use super::*;

    #[allow(dead_code)]
    fn check_parse_res(s: &str, o:&str) {
        let e = parse_str(s).unwrap();
        let t = format!("{}", e);
        assert_eq!(o, t)
    }
    #[allow(dead_code)]
    fn check_parse(s: &str) {
        let e = parse_str(s).unwrap();
        let t = format!("{}", e);
        assert_eq!(s, t)
    }

    #[allow(dead_code)]
    fn parse_fail(s: &str) {
        parse_str(s).unwrap();
    }

    #[test]
    fn test_empty() { check_parse("") }

    #[test]
    fn test_empty_qstring() { check_parse("(hello \"\")") }

    #[test]
    fn test_minimal() { check_parse("()") }

    #[test]
    fn test_string() { check_parse("hello") }

    #[test]
    fn test_qstring_a() { check_parse_res("\"hello\"", "hello") }

    #[test]
    fn test_qstring_a2() { check_parse("\"hello world\"") }

    #[test]
    fn test_qstring_a3() { check_parse("\"hello(world)\"") }

    #[test]
    fn test_number() { check_parse("1.3") }

    #[test]
    fn test_float_vs_int() { check_parse("2.0") }

    #[test]
    fn test_double() { check_parse("(())") }

    #[test]
    fn test_br_string() { check_parse("(world)") }

    #[test]
    fn test_br_qstring() { check_parse_res("(\"world\")", "(world)") }

    #[test]
    fn test_br_int() { check_parse("(42)") }

    #[test]
    fn test_br_float() { check_parse("(12.7)") }

    #[test]
    fn test_br_qbrstring() { check_parse("(\"(()\")") }

    #[test]
    fn test_number_string() { check_parse("567A_WZ") }

    #[test]
    #[should_panic(expected="called `Result::unwrap()` on an `Err` value: \"incomplete: Size(2)\"")]
    fn test_invalid1() { parse_fail("(") }

    #[test]
    #[should_panic(expected="called `Result::unwrap()` on an `Err` value: \"parse error: Alt |)|\"")]
    fn test_invalid2() { parse_fail(")") }

    #[test]
    #[should_panic(expected="called `Result::unwrap()` on an `Err` value: \"incomplete: Size(1)\"")]
    fn test_invalid3() { parse_fail("\"hello") }

    #[test]
    fn test_complex() { check_parse("(module SWITCH_3W_SIDE_MMP221-R (layer F.Cu) (descr \"\") (pad 1 thru_hole rect (size 1.2 1.2) (at -2.5 -1.6 0) (layers *.Cu *.Mask) (drill 0.8)) (pad 2 thru_hole rect (size 1.2 1.2) (at 0.0 -1.6 0) (layers *.Cu *.Mask) (drill 0.8)) (pad 3 thru_hole rect (size 1.2 1.2) (at 2.5 -1.6 0) (layers *.Cu *.Mask) (drill 0.8)) (pad 5 thru_hole rect (size 1.2 1.2) (at 0.0 1.6 0) (layers *.Cu *.Mask) (drill 0.8)) (pad 6 thru_hole rect (size 1.2 1.2) (at -2.5 1.6 0) (layers *.Cu *.Mask) (drill 0.8)) (pad 4 thru_hole rect (size 1.2 1.2) (at 2.5 1.6 0) (layers *.Cu *.Mask) (drill 0.8)) (fp_line (start -4.5 -1.75) (end 4.5 -1.75) (layer F.SilkS) (width 0.127)) (fp_line (start 4.5 -1.75) (end 4.5 1.75) (layer F.SilkS) (width 0.127)) (fp_line (start 4.5 1.75) (end -4.5 1.75) (layer F.SilkS) (width 0.127)) (fp_line (start -4.5 1.75) (end -4.5 -1.75) (layer F.SilkS) (width 0.127)))") }

    #[test]
    fn test_multiline() {
        check_parse("\
(hello \"test it\"
    (foo bar)
    (mars venus)
)
")
    }

    #[test]
    fn test_multiline2() {
        check_parse("\
(hello world
  mad
    (world)
  not)")
    }

    #[test]
    fn test_multiline_two_empty() {
        check_parse("\
(hello

world)")
    }
}

<|endoftext|>


<commit_before>// use variable names from paper, namely registers M
#![allow(non_snake_case)]

#![crate_name = "basichll"]
#![crate_type = "lib"]

/* -------------------- std libs ------------------- */
use std::cmp;
use std::fmt;
use std::hash::{ Hash, Hasher, SipHasher };
use std::iter::{ repeat, Iterator };
/* ------------------------------------------------- */

/* -------------------- helpers -------------------- */

fn alpha (m: usize) -> f64 {
    match m {
        16 => 0.673,
        32 => 0.697,
        64 => 0.709,
        _  => 0.7213/(1f64 + 1.079/(m as f64))
    }
}

fn leftmost_one_bit (v: u64) -> usize {
    let mut counter: usize = 0;
    let mut shiftable = v;

    while shiftable > 0 {
        shiftable >>= 1;
        counter += 1;
    }

    64 - counter
}

/* ------------------------------------------------- */

/* --------------- hyper log log ------------------- */

/// The `HLL` struct stores the underlying registers
/// of the HyperLogLog, with `m` many registers.
///
/// The value for `m` is derived from the desired error
/// of the estimation. The error is considered to be
/// 1.04 / sqrt(2 ^ bits). So, given a desired 1% error
/// rate, we result in around 12 bits per register,
/// producing an underlying M of 2^12 bits (i.e. 4kb).
#[derive(Clone, PartialEq)]
pub struct HLL {
    alpha: f64,
    b: u32,
    m: usize,
    M: Vec<u8>,
    // [!] this is pretty much a hack until I can think of something
    //     better to do for the monoid instance. Basically we need the
    //     zero value to always satisfy the left and right identity laws
    //     but because you need to have the number of registers be
    //     equivalent to do addition, it implies you need a distinct
    //     monoid instance per HLL configuration (i.e. an HLL of 4
    //     registers has a distinct zero from a HLL of 16).
    //
    //     Because I can't really do dependent types in rust, we instead
    //     have a distinguished zero element, that is allowed to bypass
    //     the assertions on the equivalent sizes...
    //
    //     For reference, Ed Kmett's haskell implementation actually does
    //     use higher kinded types and some tricky reflection to make the
    //     monoid instance be parameterised by a config type that is itself
    //     typed to a Peano natural encoded type. Twitter's algebird instance
    //     declares a separate class that encapsulates the notion of
    //     a "HyperLogLogMonoid" instance, and instances of that class are
    //     instantiated with the register size (i.e. the 'instance' for monoid
    //     is just an 'instance' of the class, for two interpretations of
    //     'instance').
    isZero: bool
}

impl HLL {
    /// Convenience function to produce a HLL with
    /// one hundred and twenty eight registers.
    pub fn one_hundred_twenty_eight () -> HLL {
        HLL::new(0.09192)
    }

    /// Create a new HLL with the desired standard error.
    /// Some examples might be:
    ///
    ///```ignore
    ///     bits    size    error
    ///     12      4096    0.0163
    ///     13      8192    0.0115
    ///     14      16384   0.0081
    ///```
    ///
    /// The error must be between 0.0 and 1.0. Beware using a stupidly
    /// small error size will grow beyond e.g. isize at less than 0.03
    /// (if you need something that isn't an estimate probably don't use
    /// a hyperloglog).
    pub fn new(error: f64) -> HLL {
        assert!(error > 0.0 && error < 1.0);
        // error = 1.04 / sqrt(m)
        let m = ((1.04/error) * (1.04/error)).floor() as usize;
        let b = (m as f64).log2() as u32;

        HLL {
            alpha: alpha(m),
            b: b,
            m: m,
            M: repeat(0u8).take(m).collect(),
            isZero: false
        }
    }

    /// Add an element into the hyperloglog estimate.
    /// We require the type of value to be able to be hashed.
    pub fn insert<T: Hash>(&mut self, val: &T) {
        let mut hasher = SipHasher::new();
        val.hash(&mut hasher);
        let hash = hasher.finish();

        // j is the first b many bits
        let j = (hash >> (64 - self.b)) as usize;
        // w is the remaining bits (i.e. b -> 64)
        let w = hash & (2u64.pow(64 - self.b) - 1) as u64;
        let rho = leftmost_one_bit(w) as u8;

        self.M[j] = cmp::max(self.M[j], rho);
    }

    /// Return the estimated cardinality of the observed set
    /// of elements.
    pub fn count(&self) -> f64 {
        self.range_correction(self.raw_estimate())
    }

    fn raw_estimate(&self) -> f64 {
        let sum: f64 = self.M.iter().fold (0.0, |p, &r| p + 2.0f64.powi(-(r as i32)));

        self.alpha as f64 * ((self.m  * self.m) as f64) * (1.0 / sum)
    }

    fn empty_registers (&self) -> usize {
        self.M.iter().filter(|&r| *r == 0).count()
    }

    fn range_correction(&self, e: f64) -> f64 {
        let twoTo32: f64 = 2.0f64.powi(32) as f64;

        // small range correction
        if e <= (5.0*(self.m as f64))/2.0 {
            let v  = self.empty_registers() as f64;
            let fm = self.m as f64;

            return match v {
                0.0 => e,
                _   => fm * (fm / v).ln()
            }
        }

        // medium range (no) correction
        if e <= twoTo32/30.0 {
            return e;
        }

        -twoTo32 * (1.0 - e/twoTo32).ln()
    }

    /// Access a copy of the underlying registers (this is mainly
    /// here just for debugging/testing... its unlikely you'll need
    /// this access typically).
    pub fn registers(&self) -> Vec<u8> {
        self.M.clone()
    }

    /// A completely zeroed HLL. Not particularly useful
    /// except as an identity element in `Add`.
    pub fn empty () -> HLL {
        HLL {
            alpha: 0.0,
            m: 0,
            b: 0,
            M: Vec::new(),
            isZero: true
        }
    }

}

/* ------------------------------------------------- */

/* --------------- trait instances ----------------- */

impl std::fmt::Display for HLL {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        write!(formatter, "α: {}, b: {}, m: {}", self.alpha, self.b, self.m)
    }
}

impl std::fmt::Debug for HLL {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        write!(formatter, "α: {}, b: {}, m: {}", self.alpha, self.b, self.m)
    }
}

fn zipWith<R, U: Iterator, C: Fn(U::Item, U::Item) -> R> (combo: C, left: U, right: U)
   -> Vec<R> {
    left.zip(right).map(| (l, r) | combo(l, r)).collect()
}

fn mergeRegisters (first: &Vec<u8>, second: &Vec<u8>) -> Vec<u8> {
    zipWith(|&l, &r| std::cmp::max(l, r), first.iter(), second.iter())
}

/// Adding together two HLLs produces a new HLL where
/// the larger value of each register has been selected.
impl<'a> std::ops::Add<&'a HLL> for &'a HLL {
    type Output = HLL;

    fn add(self, other: &'a HLL) -> HLL {
        // [!] gross...
        if self.isZero { return other.clone(); }
        if other.isZero { return self.clone(); }

        assert!(self.alpha == other.alpha);
        assert!(self.b == other.b);
        assert!(self.m == other.m);

        HLL {
            alpha: self.alpha,
            b: self.b,
            m: self.m,
            M: mergeRegisters(&self.M, &other.M),
            isZero: false
        }
    }
}

/* ------------------------------------------------- */
<commit_msg>Add back cmp::Eq to HLL<commit_after>// use variable names from paper, namely registers M
#![allow(non_snake_case)]

#![crate_name = "basichll"]
#![crate_type = "lib"]

/* -------------------- std libs ------------------- */
use std::cmp;
use std::fmt;
use std::hash::{ Hash, Hasher, SipHasher };
use std::iter::{ repeat, Iterator };
/* ------------------------------------------------- */

/* -------------------- helpers -------------------- */

fn alpha (m: usize) -> f64 {
    match m {
        16 => 0.673,
        32 => 0.697,
        64 => 0.709,
        _  => 0.7213/(1f64 + 1.079/(m as f64))
    }
}

fn leftmost_one_bit (v: u64) -> usize {
    let mut counter: usize = 0;
    let mut shiftable = v;

    while shiftable > 0 {
        shiftable >>= 1;
        counter += 1;
    }

    64 - counter
}

/* ------------------------------------------------- */

/* --------------- hyper log log ------------------- */

/// The `HLL` struct stores the underlying registers
/// of the HyperLogLog, with `m` many registers.
///
/// The value for `m` is derived from the desired error
/// of the estimation. The error is considered to be
/// 1.04 / sqrt(2 ^ bits). So, given a desired 1% error
/// rate, we result in around 12 bits per register,
/// producing an underlying M of 2^12 bits (i.e. 4kb).
#[derive(Clone, PartialEq)]
pub struct HLL {
    alpha: f64,
    b: u32,
    m: usize,
    M: Vec<u8>,
    // [!] this is pretty much a hack until I can think of something
    //     better to do for the monoid instance. Basically we need the
    //     zero value to always satisfy the left and right identity laws
    //     but because you need to have the number of registers be
    //     equivalent to do addition, it implies you need a distinct
    //     monoid instance per HLL configuration (i.e. an HLL of 4
    //     registers has a distinct zero from a HLL of 16).
    //
    //     Because I can't really do dependent types in rust, we instead
    //     have a distinguished zero element, that is allowed to bypass
    //     the assertions on the equivalent sizes...
    //
    //     For reference, Ed Kmett's haskell implementation actually does
    //     use higher kinded types and some tricky reflection to make the
    //     monoid instance be parameterised by a config type that is itself
    //     typed to a Peano natural encoded type. Twitter's algebird instance
    //     declares a separate class that encapsulates the notion of
    //     a "HyperLogLogMonoid" instance, and instances of that class are
    //     instantiated with the register size (i.e. the 'instance' for monoid
    //     is just an 'instance' of the class, for two interpretations of
    //     'instance').
    isZero: bool
}

impl std::cmp::Eq for HLL {}

impl HLL {
    /// Convenience function to produce a HLL with
    /// one hundred and twenty eight registers.
    pub fn one_hundred_twenty_eight () -> HLL {
        HLL::new(0.09192)
    }

    /// Create a new HLL with the desired standard error.
    /// Some examples might be:
    ///
    ///```ignore
    ///     bits    size    error
    ///     12      4096    0.0163
    ///     13      8192    0.0115
    ///     14      16384   0.0081
    ///```
    ///
    /// The error must be between 0.0 and 1.0. Beware using a stupidly
    /// small error size will grow beyond e.g. isize at less than 0.03
    /// (if you need something that isn't an estimate probably don't use
    /// a hyperloglog).
    pub fn new(error: f64) -> HLL {
        assert!(error > 0.0 && error < 1.0);
        // error = 1.04 / sqrt(m)
        let m = ((1.04/error) * (1.04/error)).floor() as usize;
        let b = (m as f64).log2() as u32;

        HLL {
            alpha: alpha(m),
            b: b,
            m: m,
            M: repeat(0u8).take(m).collect(),
            isZero: false
        }
    }

    /// Add an element into the hyperloglog estimate.
    /// We require the type of value to be able to be hashed.
    pub fn insert<T: Hash>(&mut self, val: &T) {
        let mut hasher = SipHasher::new();
        val.hash(&mut hasher);
        let hash = hasher.finish();

        // j is the first b many bits
        let j = (hash >> (64 - self.b)) as usize;
        // w is the remaining bits (i.e. b -> 64)
        let w = hash & (2u64.pow(64 - self.b) - 1) as u64;
        let rho = leftmost_one_bit(w) as u8;

        self.M[j] = cmp::max(self.M[j], rho);
    }

    /// Return the estimated cardinality of the observed set
    /// of elements.
    pub fn count(&self) -> f64 {
        self.range_correction(self.raw_estimate())
    }

    fn raw_estimate(&self) -> f64 {
        let sum: f64 = self.M.iter().fold (0.0, |p, &r| p + 2.0f64.powi(-(r as i32)));

        self.alpha as f64 * ((self.m  * self.m) as f64) * (1.0 / sum)
    }

    fn empty_registers (&self) -> usize {
        self.M.iter().filter(|&r| *r == 0).count()
    }

    fn range_correction(&self, e: f64) -> f64 {
        let twoTo32: f64 = 2.0f64.powi(32) as f64;

        // small range correction
        if e <= (5.0*(self.m as f64))/2.0 {
            let v  = self.empty_registers() as f64;
            let fm = self.m as f64;

            return match v {
                0.0 => e,
                _   => fm * (fm / v).ln()
            }
        }

        // medium range (no) correction
        if e <= twoTo32/30.0 {
            return e;
        }

        -twoTo32 * (1.0 - e/twoTo32).ln()
    }

    /// Access a copy of the underlying registers (this is mainly
    /// here just for debugging/testing... its unlikely you'll need
    /// this access typically).
    pub fn registers(&self) -> Vec<u8> {
        self.M.clone()
    }

    /// A completely zeroed HLL. Not particularly useful
    /// except as an identity element in `Add`.
    pub fn empty () -> HLL {
        HLL {
            alpha: 0.0,
            m: 0,
            b: 0,
            M: Vec::new(),
            isZero: true
        }
    }

}

/* ------------------------------------------------- */

/* --------------- trait instances ----------------- */

impl std::fmt::Display for HLL {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        write!(formatter, "α: {}, b: {}, m: {}", self.alpha, self.b, self.m)
    }
}

impl std::fmt::Debug for HLL {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        write!(formatter, "α: {}, b: {}, m: {}", self.alpha, self.b, self.m)
    }
}

fn zipWith<R, U: Iterator, C: Fn(U::Item, U::Item) -> R> (combo: C, left: U, right: U)
   -> Vec<R> {
    left.zip(right).map(| (l, r) | combo(l, r)).collect()
}

fn mergeRegisters (first: &Vec<u8>, second: &Vec<u8>) -> Vec<u8> {
    zipWith(|&l, &r| std::cmp::max(l, r), first.iter(), second.iter())
}

/// Adding together two HLLs produces a new HLL where
/// the larger value of each register has been selected.
impl<'a> std::ops::Add<&'a HLL> for &'a HLL {
    type Output = HLL;

    fn add(self, other: &'a HLL) -> HLL {
        // [!] gross...
        if self.isZero { return other.clone(); }
        if other.isZero { return self.clone(); }

        assert!(self.alpha == other.alpha);
        assert!(self.b == other.b);
        assert!(self.m == other.m);

        HLL {
            alpha: self.alpha,
            b: self.b,
            m: self.m,
            M: mergeRegisters(&self.M, &other.M),
            isZero: false
        }
    }
}

/* ------------------------------------------------- */
<|endoftext|>


<commit_before>extern crate rand;
use HaltCondition::{ Epochs, MSE };
use UpdateRule::{ Stochastic, Batch };
use std::iter::{Zip, Enumerate};
use std::slice;
use std::num::Float;

#[test]
fn it_works() {
}

type Node = Vec<f64>;
type Layer = Vec<Node>;

#[derive(Debug)]
pub enum HaltCondition {
    Epochs(usize),
    MSE(f64),
}

#[derive(Debug)]
pub enum UpdateRule {
    Stochastic,
    Batch
}

#[derive(Debug)]
struct Trainer {
    rate: f64,
    momentum: f64,
    log_interval: Option<usize>,
    halt_condition: HaltCondition,
    update_rule: UpdateRule,
    threads: usize,
}

#[derive(Debug)]
#[derive(Clone)]
pub struct NN {
    layers: Vec<Layer>,
    num_inputs: usize,
}

impl NN {
    fn train(&self) -> Trainer {
        unimplemented!()
    }

    // returns a results with ok being the error rate of the network found at the end and
    // Err is a string with an explaination
    fn train_option(&mut self, examples: &[(&Vec<f64>, &Vec<f64>)], rate: f64, momentum: f64, log_interval: Option<usize>,
                    halt_condition: HaltCondition, update_rule: UpdateRule, threads: usize) -> Result<usize, &str> {

        let mut epochs = 0;
        let mut error_rate = 0.0;

        loop {

            let mut prev_deltas = make_weights_tracker(&self.layers, 0.0f64);
            let mut batch_data = make_weights_tracker(&self.layers, 0.0f64);
            let mut error_sum: f64 = 0.0;

            for &(inputs, targets) in examples.iter() {
                let results = do_run(self, inputs);
                let weight_updates = calculate_weight_update_info(self, &results, &targets);
                error_sum += calculate_error(&results, targets);
                match update_rule {
                    Batch => update_batch_data(&mut batch_data, &weight_updates),
                    Stochastic => update_weights(self, &weight_updates, &mut prev_deltas, rate, momentum),
                }
            }

            // if we're in batch mode, update the weights now
            match update_rule {
                Batch => update_weights(self, &batch_data, &mut prev_deltas, rate, momentum),
                Stochastic => (),
            }

            epochs += 1;

            // log error rate if neccessary
            match log_interval {
                Some(interval) if epochs % interval == 0 => {
                    println!("error rate: {}", error_rate);
                },
                _ => (),
            }

            // check if we've met the halt condition yet
            match halt_condition {
                Epochs(epochs_halt) => {
                    if epochs == epochs_halt { break; }
                },
                MSE(target_error) => {
                    if target_error == error_rate { break; }
                }
            }
        }

        unimplemented!();
    }

    fn run(&self, inputs: &Vec<f64>) -> Vec<f64>{
        unimplemented!()
    }
}

fn do_run(nn: &NN, inputs: &Vec<f64>) -> Vec<Vec<f64>>{
    let mut results = Vec::new();
    results.push(inputs.clone());
    let mut prev_layer_results = inputs;
    for layer in nn.layers.iter() {
        let mut layer_results = Vec::new();
        for node in layer.iter() {
            layer_results.push( sigmoid(modified_dotprod(&node, &prev_layer_results)) )
        }
        results.push(layer_results);
    }
    results
}

fn modified_dotprod(node: &Node, values: &Vec<f64>) -> f64 {
    let mut total = node[0]; // for the threshold weight
    for (weight, value) in node.iter().skip(1).zip(values.iter()) {
        total += weight * value;
    }
    total
}

fn sigmoid(y: f64) -> f64 {
    1.0 / (1.0 + Float::exp(-y))
}

fn update_batch_data(batch_data: &mut Vec<Vec<Vec<f64>>> , weight_updates: &Vec<Vec<Vec<f64>>>) {
    unimplemented!()
}

fn update_weights(nn: &mut NN, network_weight_updates: &Vec<Vec<Vec<f64>>>, prev_deltas: &mut Vec<Vec<Vec<f64>>>, rate: f64, momentum: f64) {
    for layer_index in 0..nn.layers.len() {
        let mut layer = &mut nn.layers[layer_index];
        let layer_weight_updates = &network_weight_updates[layer_index];
        for node_index in 0..layer.len() {
            let mut node = &mut layer[node_index];
            let node_weight_updates = &layer_weight_updates[node_index];
            for weight_index in 0..node.len() {
                let weight = node[weight_index];
                let weight_update = node_weight_updates[weight_index];
                let prev_delta = prev_deltas[layer_index][node_index][weight_index];
                let delta = (rate * weight_update) + (momentum * prev_delta);
                node[weight_index] += delta;
                prev_deltas[layer_index][node_index][weight_index] = delta;
            }
        }
    }
}

fn calculate_weight_update_info(nn: &NN, results: &Vec<Vec<f64>>, targets: &[f64]) -> Vec<Vec<Vec<f64>>> {
    let mut network_errors = Vec::new();
    let mut network_weight_updates = Vec::new();

    let mut next_layer_errors: Vec<f64> = Vec::new(); // TODO: find a way to not do this

    let layers = &nn.layers;
    let network_results = &results[1..]; // skip the input layer

    for (layer_index, (layer_nodes, layer_results)) in iterZipEnum(layers, network_results) {
        let prev_layer_results = &results[layer_index];
        let mut layer_errors = Vec::new();
        let mut layer_weight_updates = Vec::new();

        let next_layer_nodes: &Layer = &&layers[layer_index+1];

        for (node_index, (node, &result)) in iterZipEnum(layer_nodes, layer_results) {
            let mut node_weight_updates = Vec::new();
            let mut node_error = 0f64;

            // calculate error for this node
            if layer_index == layers.len() - 1 {
                node_error = result * (1f64 - result) * (targets[node_index] - result);
            } else {
                let mut sum = 0f64;
                for (next_node, &next_node_error_data) in next_layer_nodes.iter().zip(next_layer_errors.iter()) {
                    sum += next_node[node_index+1] * next_node_error_data; // +1 because the 0th weight is the threshold
                }
                node_error = result * (1f64 - result) * sum;
            }

            // calculate weight updates for this node
            for weight_index in 0..node.len() {
                let mut prev_layer_result = 1f64;
                if weight_index == 0 {
                    prev_layer_result = 1.0; // theshold
                } else {
                    prev_layer_result = prev_layer_results[weight_index-1];
                }
                let weight_update = node_error * prev_layer_result;
                node_weight_updates.push(weight_update);
            }

            layer_errors.push(node_error);
            layer_weight_updates.push(node_weight_updates);
        }

        next_layer_errors = layer_errors.clone();

        network_errors.push(layer_errors);
        network_weight_updates.push(layer_weight_updates);
    }

    // updates were build by backpropagation so reverse them
    network_weight_updates.reverse();

    network_weight_updates
}

fn iterZipEnum<'s, 't, S: 's, T: 't>(s: &'s [S], t: &'t [T]) ->
    Enumerate<Zip<slice::Iter<'s, S>, slice::Iter<'t, T>>>  {
    s.iter().zip(t.iter()).enumerate()
}

fn calculate_error(results: &Vec<Vec<f64>>, targets: &[f64]) -> f64 {
    unimplemented!()
}

fn make_weights_tracker<T: Clone>(layers: &Vec<Layer>, place_holder: T) -> Vec<Vec<Vec<T>>> {
    let mut network_level = Vec::new();
    for layer in layers.iter() {
        let mut layer_level = Vec::new();
        for node in layer.iter() {
            let mut node_level = Vec::new();
            for weight in node.iter() {
                node_level.push(place_holder.clone());
            }
            layer_level.push(node_level);
        }
        network_level.push(layer_level);
    }

    network_level
}

fn new_nn(layers_sizes: &[usize]) -> NN {
        if layers_sizes.len() < 2 {
            panic!("must have at least two layers");
        }
        let mut layers = Vec::new();
        let mut it = layers_sizes.iter();
        // get the first layer size
        let first_layer_size = *it.next().unwrap();

        // setup the rest of the layers
        let mut prev_layer_size = first_layer_size;
        for &layer_size in it {
            let mut layer: Layer = Vec::new();
            for _ in 0..layer_size {
                let mut node: Node = vec![rand::random(); prev_layer_size];
                node.shrink_to_fit();
                layer.push(node)
            }
            layer.shrink_to_fit();
            layers.push(layer);
            prev_layer_size = layer_size;
        }
        layers.shrink_to_fit();
        NN { layers: layers, num_inputs: first_layer_size }
    }<commit_msg>implemented Trainer<commit_after>extern crate rand;
use HaltCondition::{ Epochs, MSE };
use UpdateRule::{ Stochastic, Batch };
use std::iter::{Zip, Enumerate};
use std::slice;
use std::num::Float;

#[test]
fn it_works() {
}

type Node = Vec<f64>;
type Layer = Vec<Node>;

#[derive(Debug)]
#[derive(Clone)]
pub enum HaltCondition {
    Epochs(usize),
    MSE(f64),
}

#[derive(Debug)]
#[derive(Clone)]
pub enum UpdateRule {
    Stochastic,
    Batch
}

static DEFAULT_LEARNING_RATE: f64 = 0.3f64;
static DEFAULT_MOMENTUM: f64 = 0f64;
static DEFAULT_EPOCHS: usize = 1000;

#[derive(Debug)]
struct Trainer<'a> {
    rate: f64,
    momentum: f64,
    log_interval: Option<usize>,
    halt_condition: HaltCondition,
    update_rule: UpdateRule,
    threads: usize,
    nn: &'a mut NN,
}

impl<'a> Trainer<'a>  {

    fn rate(&mut self, rate: f64) -> &mut Trainer<'a> {
        self.rate = rate;
        self
    }

    fn momentum(&mut self, momentum: f64) -> &mut Trainer<'a> {
        self.momentum = momentum;
        self
    }

    fn log_interval(&mut self, log_interval: Option<usize>) -> &mut Trainer<'a> {
        self.log_interval = log_interval;
        self
    }

    fn halt_condition(&mut self, halt_condition: HaltCondition) -> &mut Trainer<'a> {
        self.halt_condition = halt_condition;
        self
    }

    fn update_rule(&mut self, update_rule: UpdateRule) -> &mut Trainer<'a> {
        self.update_rule = update_rule;
        self
    }

    fn threads(&mut self, threads: usize) -> &mut Trainer<'a> {
        self.threads = threads;
        self
    }

    fn go(&mut self, examples: &[(&Vec<f64>, &Vec<f64>)]) -> Result<usize, &str> {
        self.nn.train_option(
            examples,
            self.rate,
            self.momentum,
            self.log_interval,
            self.halt_condition.clone(),
            self.update_rule.clone(),
            self.threads
        )
    }

}

#[derive(Debug)]
#[derive(Clone)]
pub struct NN {
    layers: Vec<Layer>,
    num_inputs: usize,
}

impl NN {
    fn train(&mut self) -> Trainer {
        Trainer {
            rate: DEFAULT_LEARNING_RATE,
            momentum: DEFAULT_MOMENTUM,
            log_interval: None,
            halt_condition: Epochs(DEFAULT_EPOCHS),
            update_rule: Stochastic,
            threads: 1,
            nn: self
        }
        // unimplemented!()
    }

    // returns a results with ok being the error rate of the network found at the end and
    // Err is a string with an explaination
    fn train_option(&mut self, examples: &[(&Vec<f64>, &Vec<f64>)], rate: f64, momentum: f64, log_interval: Option<usize>,
                    halt_condition: HaltCondition, update_rule: UpdateRule, threads: usize) -> Result<usize, &str> {

        let mut epochs = 0;
        let mut error_rate = 0.0;

        loop {

            let mut prev_deltas = make_weights_tracker(&self.layers, 0.0f64);
            let mut batch_data = make_weights_tracker(&self.layers, 0.0f64);
            let mut error_sum: f64 = 0.0;

            for &(inputs, targets) in examples.iter() {
                let results = do_run(self, inputs);
                let weight_updates = calculate_weight_update_info(self, &results, &targets);
                error_sum += calculate_error(&results, targets);
                match update_rule {
                    Batch => update_batch_data(&mut batch_data, &weight_updates),
                    Stochastic => update_weights(self, &weight_updates, &mut prev_deltas, rate, momentum),
                }
            }

            // if we're in batch mode, update the weights now
            match update_rule {
                Batch => update_weights(self, &batch_data, &mut prev_deltas, rate, momentum),
                Stochastic => (),
            }

            epochs += 1;

            // log error rate if neccessary
            match log_interval {
                Some(interval) if epochs % interval == 0 => {
                    println!("error rate: {}", error_rate);
                },
                _ => (),
            }

            // check if we've met the halt condition yet
            match halt_condition {
                Epochs(epochs_halt) => {
                    if epochs == epochs_halt { break; }
                },
                MSE(target_error) => {
                    if target_error == error_rate { break; }
                }
            }
        }

        unimplemented!();
    }

    fn run(&self, inputs: &Vec<f64>) -> Vec<f64> {
        do_run(self, inputs).pop().unwrap()
    }
}

fn do_run(nn: &NN, inputs: &Vec<f64>) -> Vec<Vec<f64>> {
    let mut results = Vec::new();
    results.push(inputs.clone());
    let mut prev_layer_results = inputs;
    for layer in nn.layers.iter() {
        let mut layer_results = Vec::new();
        for node in layer.iter() {
            layer_results.push( sigmoid(modified_dotprod(&node, &prev_layer_results)) )
        }
        results.push(layer_results);
    }
    results
}

fn modified_dotprod(node: &Node, values: &Vec<f64>) -> f64 {
    let mut total = node[0]; // for the threshold weight
    for (weight, value) in node.iter().skip(1).zip(values.iter()) {
        total += weight * value;
    }
    total
}

fn sigmoid(y: f64) -> f64 {
    1.0 / (1.0 + Float::exp(-y))
}

fn update_batch_data(batch_data: &mut Vec<Vec<Vec<f64>>> , weight_updates: &Vec<Vec<Vec<f64>>>) {
    unimplemented!()
}

fn update_weights(nn: &mut NN, network_weight_updates: &Vec<Vec<Vec<f64>>>, prev_deltas: &mut Vec<Vec<Vec<f64>>>, rate: f64, momentum: f64) {
    for layer_index in 0..nn.layers.len() {
        let mut layer = &mut nn.layers[layer_index];
        let layer_weight_updates = &network_weight_updates[layer_index];
        for node_index in 0..layer.len() {
            let mut node = &mut layer[node_index];
            let node_weight_updates = &layer_weight_updates[node_index];
            for weight_index in 0..node.len() {
                let weight = node[weight_index];
                let weight_update = node_weight_updates[weight_index];
                let prev_delta = prev_deltas[layer_index][node_index][weight_index];
                let delta = (rate * weight_update) + (momentum * prev_delta);
                node[weight_index] += delta;
                prev_deltas[layer_index][node_index][weight_index] = delta;
            }
        }
    }
}

fn calculate_weight_update_info(nn: &NN, results: &Vec<Vec<f64>>, targets: &[f64]) -> Vec<Vec<Vec<f64>>> {
    let mut network_errors = Vec::new();
    let mut network_weight_updates = Vec::new();

    let mut next_layer_errors: Vec<f64> = Vec::new(); // TODO: find a way to not do this

    let layers = &nn.layers;
    let network_results = &results[1..]; // skip the input layer

    for (layer_index, (layer_nodes, layer_results)) in iterZipEnum(layers, network_results) {
        let prev_layer_results = &results[layer_index];
        let mut layer_errors = Vec::new();
        let mut layer_weight_updates = Vec::new();

        let next_layer_nodes: &Layer = &&layers[layer_index+1];

        for (node_index, (node, &result)) in iterZipEnum(layer_nodes, layer_results) {
            let mut node_weight_updates = Vec::new();
            let mut node_error = 0f64;

            // calculate error for this node
            if layer_index == layers.len() - 1 {
                node_error = result * (1f64 - result) * (targets[node_index] - result);
            } else {
                let mut sum = 0f64;
                for (next_node, &next_node_error_data) in next_layer_nodes.iter().zip(next_layer_errors.iter()) {
                    sum += next_node[node_index+1] * next_node_error_data; // +1 because the 0th weight is the threshold
                }
                node_error = result * (1f64 - result) * sum;
            }

            // calculate weight updates for this node
            for weight_index in 0..node.len() {
                let mut prev_layer_result = 1f64;
                if weight_index == 0 {
                    prev_layer_result = 1.0; // theshold
                } else {
                    prev_layer_result = prev_layer_results[weight_index-1];
                }
                let weight_update = node_error * prev_layer_result;
                node_weight_updates.push(weight_update);
            }

            layer_errors.push(node_error);
            layer_weight_updates.push(node_weight_updates);
        }

        next_layer_errors = layer_errors.clone();

        network_errors.push(layer_errors);
        network_weight_updates.push(layer_weight_updates);
    }

    // updates were build by backpropagation so reverse them
    network_weight_updates.reverse();

    network_weight_updates
}

fn iterZipEnum<'s, 't, S: 's, T: 't>(s: &'s [S], t: &'t [T]) ->
    Enumerate<Zip<slice::Iter<'s, S>, slice::Iter<'t, T>>>  {
    s.iter().zip(t.iter()).enumerate()
}

fn calculate_error(results: &Vec<Vec<f64>>, targets: &[f64]) -> f64 {
    unimplemented!()
}

fn make_weights_tracker<T: Clone>(layers: &Vec<Layer>, place_holder: T) -> Vec<Vec<Vec<T>>> {
    let mut network_level = Vec::new();
    for layer in layers.iter() {
        let mut layer_level = Vec::new();
        for node in layer.iter() {
            let mut node_level = Vec::new();
            for weight in node.iter() {
                node_level.push(place_holder.clone());
            }
            layer_level.push(node_level);
        }
        network_level.push(layer_level);
    }

    network_level
}

fn new_nn(layers_sizes: &[usize]) -> NN {
        if layers_sizes.len() < 2 {
            panic!("must have at least two layers");
        }
        let mut layers = Vec::new();
        let mut it = layers_sizes.iter();
        // get the first layer size
        let first_layer_size = *it.next().unwrap();

        // setup the rest of the layers
        let mut prev_layer_size = first_layer_size;
        for &layer_size in it {
            let mut layer: Layer = Vec::new();
            for _ in 0..layer_size {
                let mut node: Node = vec![rand::random(); prev_layer_size];
                node.shrink_to_fit();
                layer.push(node)
            }
            layer.shrink_to_fit();
            layers.push(layer);
            prev_layer_size = layer_size;
        }
        layers.shrink_to_fit();
        NN { layers: layers, num_inputs: first_layer_size }
    }<|endoftext|>