<commit_before>// Copyright 2017 pdb Developers
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.

use common::*;
use source::*;

mod page_list;
use self::page_list::PageList;

use std::fmt;

type PageNumber = u32;

#[derive(Debug,Copy,Clone)]
struct Header {
    page_size: usize,
    maximum_valid_page_number: PageNumber,
}

impl Header {
    fn pages_needed_to_store(&self, bytes: usize) -> usize {
        (bytes + (self.page_size - 1)) / self.page_size
    }

    fn validate_page_number(&self, page_number: u32) -> Result<PageNumber> {
        if page_number == 0 || page_number > self.maximum_valid_page_number {
            Err(Error::PageReferenceOutOfRange(page_number))
        } else {
            Ok(page_number as PageNumber)
        }
    }
}

/// Represents a stream table at various stages of access
#[doc(hidden)]
#[derive(Debug)]
enum StreamTable<'s> {
    /// The MSF header gives us the size of the table in bytes, and the list of pages (usually one)
    /// where we can find the list of pages that contain the stream table.
    HeaderOnly { size_in_bytes: usize, stream_table_location_location: PageList },

    /// Given the HeaderOnly information, we can do an initial read to get the actual location of
    /// the stream table as a PageList.
    TableFound { stream_table_location: PageList },

    // Given the table location, we can access the stream table itself
    Available { stream_table_view: Box<SourceView<'s>> }
}

fn view<'s>(source: &mut Source<'s>, page_list: &PageList) -> Result<Box<SourceView<'s>>> {
    // view it
    let view = source.view(page_list.source_slices())?;

    // double check our Source
    // if the Source didn't return the requested bits, that's an implementation bug, so
    // assert instead of returning an error
    assert_eq!(view.as_slice().len(), page_list.len());

    // done
    Ok(view)
}

mod big {
    use super::*;

    pub const MAGIC: &'static [u8] = b"Microsoft C/C++ MSF 7.00\r\n\x1a\x44\x53\x00\x00\x00";

    /// The PDB header as stored on disk.
    /// See the Microsoft code for reference: https://github.com/Microsoft/microsoft-pdb/blob/082c5290e5aff028ae84e43affa8be717aa7af73/PDB/msf/msf.cpp#L946
    #[derive(Debug, Pread)]
    #[repr(C, packed)]
    struct RawHeader {
        magic: [u8; 32],
        page_size: u32,
        free_page_map: u32,
        pages_used: u32,
        directory_size: u32,
        _reserved: u32,
    }

    #[derive(Debug)]
    pub struct BigMSF<'s, S> {
        header: Header,
        source: S,
        stream_table: StreamTable<'s>,
    }

    impl<'s, S: Source<'s>> BigMSF<'s, S> {
        pub fn new(source: S, header_view: Box<SourceView>) -> Result<BigMSF<'s, S>> {
            let mut buf = ParseBuffer::from(header_view.as_slice());
            let header: RawHeader = buf.parse()?;

            if header.magic != MAGIC {
                return Err(Error::UnrecognizedFileFormat);
            }

            if header.page_size.count_ones() != 1 || header.page_size < 0x100
                || header.page_size > (128 * 0x10000) {
                    return Err(Error::InvalidPageSize(header.page_size));
                }

            let header_object = Header{
                page_size: header.page_size as usize,
                maximum_valid_page_number: header.pages_used,
            };

            // calculate how many pages are needed to store the stream table
            let size_of_stream_table_in_pages = header_object.pages_needed_to_store(header.directory_size as usize);

            // now: how many pages are needed to store the list of pages that store the stream table?
            // each page entry is a u32, so multiply by four
            let size_of_stream_table_page_list_in_pages = header_object.pages_needed_to_store(size_of_stream_table_in_pages * 4);

            // read the list of stream table page list pages, which immediately follow the header
            // yes, this is a stupid level of indirection
            let mut stream_table_page_list_page_list = PageList::new(header_object.page_size);
            for _ in 0..size_of_stream_table_page_list_in_pages {
                let n = buf.parse_u32()?;
                stream_table_page_list_page_list.push(header_object.validate_page_number(n)?);
            }

            // truncate the stream table location location to the correct size
            stream_table_page_list_page_list.truncate(size_of_stream_table_in_pages * 4);

            Ok(BigMSF{
                header: header_object,
                source: source,
                stream_table: StreamTable::HeaderOnly {
                    size_in_bytes: header.directory_size as usize,
                    stream_table_location_location: stream_table_page_list_page_list,
                },
            })
        }

        fn find_stream_table(&mut self) -> Result<()> {
            let mut new_stream_table: Option<StreamTable> = None;

            if let StreamTable::HeaderOnly { size_in_bytes, ref stream_table_location_location } = self.stream_table {
                // the header indicated we need to read size_in_pages page numbers from the
                // specified PageList.

                // ask to view the location location
                let location_location = view(&mut self.source, stream_table_location_location)?;

                // build a PageList
                let mut page_list = PageList::new(self.header.page_size);
                let mut buf = ParseBuffer::from(location_location.as_slice());
                while buf.len() > 0 {
                    let n = buf.parse_u32()?;
                    page_list.push(self.header.validate_page_number(n)?);
                }

                page_list.truncate(size_in_bytes);

                // remember what we learned
                new_stream_table = Some(StreamTable::TableFound {
                    stream_table_location: page_list,
                });
            }

            if let Some(st) = new_stream_table {
                self.stream_table = st;
            }

            Ok(())
        }

        fn make_stream_table_available(&mut self) -> Result<()> {
            // do the initial read if we must
            if let StreamTable::HeaderOnly { .. } = self.stream_table {
                self.find_stream_table()?;
            }

            // do we need to map the stream table itself?
            let mut new_stream_table: Option<StreamTable> = None;
            if let StreamTable::TableFound { ref stream_table_location } = self.stream_table {
                // yep
                // ask the source to view it
                let stream_table_view = view(&mut self.source, stream_table_location)?;

                // done
                new_stream_table = Some(StreamTable::Available {
                    stream_table_view: stream_table_view,
                });
            }

            if let Some(st) = new_stream_table {
                self.stream_table = st;
            }

            // stream table is available
            assert!(match &self.stream_table {
                &StreamTable::Available { .. } => true,
                _ => false
            });

            Ok(())
        }

        fn look_up_stream(&mut self, stream_number: u32) -> Result<PageList> {
            // ensure the stream table is available
            self.make_stream_table_available()?;

            let header = self.header;

            // declare the things we're going to find
            let bytes_in_stream: u32;
            let page_list: PageList;

            if let StreamTable::Available { ref stream_table_view } = self.stream_table {
                let stream_table_slice = stream_table_view.as_slice();
                let mut stream_table = ParseBuffer::from(stream_table_slice);

                // the stream table is structured as:
                // stream_count
                // 0..stream_count: size of stream in bytes (0xffffffff indicating "stream does not exist")
                // stream 0: PageNumber
                // stream 1: PageNumber, PageNumber
                // stream 2: PageNumber, PageNumber, PageNumber, PageNumber, PageNumber
                // stream 3: PageNumber, PageNumber, PageNumber, PageNumber
                // (number of pages determined by number of bytes)

                let stream_count = stream_table.parse_u32()?;

                // check if we've already outworn our welcome
                if stream_number >= stream_count {
                    return Err(Error::StreamNotFound(stream_number))
                }

                // we now have {stream_count} u32s describing the length of each stream

                // walk over the streams before the requested stream
                // we need to pay attention to how big each one is, since their page numbers come
                // before our page numbers in the stream table
                let mut page_numbers_to_skip: usize = 0;
                for _ in 0..stream_number {
                    let bytes = stream_table.parse_u32()?;
                    if bytes == 0xffffffff {
                        // stream is not present, ergo nothing to skip
                    } else {
                        page_numbers_to_skip += header.pages_needed_to_store(bytes as usize);
                    }
                }

                // read our stream's size
                bytes_in_stream = stream_table.parse_u32()?;
                if bytes_in_stream == 0xffffffff {
                    return Err(Error::StreamNotFound(stream_number))
                }
                let pages_in_stream = header.pages_needed_to_store(bytes_in_stream as usize);

                // skip the remaining streams' byte counts
                let _ = stream_table.take((stream_count - stream_number - 1) as usize * 4)?;

                // skip the preceding streams' page numbers
                let _ = stream_table.take((page_numbers_to_skip as usize) * 4)?;

                // we're now at the list of pages for our stream
                // accumulate them into a PageList
                let mut list = PageList::new(header.page_size);
                for _ in 0..pages_in_stream {
                    let page_number = stream_table.parse_u32()?;
                    list.push(self.header.validate_page_number(page_number)?);
                }

                // truncate to the size of the stream
                list.truncate(bytes_in_stream as usize);

                page_list = list;
            } else {
                unreachable!();
            }

            // done!
            Ok(page_list)
        }
    }

    impl<'s, S: Source<'s>> MSF<'s, S> for BigMSF<'s, S> {
        fn get(&mut self, stream_number: u32, limit: Option<usize>) -> Result<Stream<'s>> {
            // look up the stream
            let mut page_list = self.look_up_stream(stream_number)?;

            // apply any limits we have
            if let Some(limit) = limit {
                page_list.truncate(limit);
            }

            // now that we know where this stream lives, we can view it
            let view = view(&mut self.source, &page_list)?;

            // pack it into a Stream
            let stream = Stream {
                source_view: view,
            };

            Ok(stream)
        }
    }
}

mod small {
    pub const MAGIC: &'static [u8] = b"Microsoft C/C++ program database 2.00\r\n\x1a\x4a\x47";
    // TODO: implement SmallMSF
}

/// Represents a single Stream within the multi-stream file.
#[derive(Debug)]
pub struct Stream<'s> {
    source_view: Box<SourceView<'s>>,
}

impl<'s> Stream<'s> {
    #[inline]
    pub fn parse_buffer(&self) -> ParseBuffer {
        let slice = self.source_view.as_slice();
        ParseBuffer::from(slice)
    }
}

/// Provides access to a "multi-stream file", which is the container format used by PDBs.
pub trait MSF<'s, S> : fmt::Debug {
    /// Accesses a stream by stream number, optionally restricted by a byte limit.
    fn get(&mut self, stream_number: u32, limit: Option<usize>) -> Result<Stream<'s>>;
}

fn header_matches(actual: &[u8], expected: &[u8]) -> bool {
    actual.len() >= expected.len() && &actual[0..expected.len()] == expected
}

pub fn open_msf<'s, S: Source<'s> + 's>(mut source: S) -> Result<Box<MSF<'s, S> + 's>> {
    // map the header
    let mut header_location = PageList::new(4096);
    header_location.push(0);
    let header_view = view(&mut source, &header_location)?;

    // see if it's a BigMSF
    if header_matches(header_view.as_slice(), big::MAGIC) {
        // claimed!
        let bigmsf = big::BigMSF::new(source, header_view)?;
        return Ok(Box::new(bigmsf))
    }

    if header_matches(header_view.as_slice(), small::MAGIC) {
        // sorry
        return Err(Error::UnimplementedFeature("small MSF file format"));
    }

    Err(Error::UnrecognizedFileFormat)
}

#[cfg(test)]
mod tests {

    mod header {
        use msf::Header;
        use common::Error;

        #[test]
        fn test_pages_needed_to_store() {
            let h = Header{
                page_size: 4096,
                maximum_valid_page_number: 15,
            };
            assert_eq!(h.pages_needed_to_store(0), 0);
            assert_eq!(h.pages_needed_to_store(1), 1);
            assert_eq!(h.pages_needed_to_store(1024), 1);
            assert_eq!(h.pages_needed_to_store(2048), 1);
            assert_eq!(h.pages_needed_to_store(4095), 1);
            assert_eq!(h.pages_needed_to_store(4096), 1);
            assert_eq!(h.pages_needed_to_store(4097), 2);
        }

        #[test]
        fn test_validate_page_number() {
            let h = Header{
                page_size: 4096,
                maximum_valid_page_number: 15,
            };
            assert!(match h.validate_page_number(0) { Err(Error::PageReferenceOutOfRange(0)) => true, _ => false });
            assert!(match h.validate_page_number(1) { Ok(1) => true, _ => false });
            assert!(match h.validate_page_number(2) { Ok(2) => true, _ => false });
            assert!(match h.validate_page_number(14) { Ok(14) => true, _ => false });
            assert!(match h.validate_page_number(15) { Ok(15) => true, _ => false });
            assert!(match h.validate_page_number(16) { Err(Error::PageReferenceOutOfRange(16)) => true, _ => false });
            assert!(match h.validate_page_number(17) { Err(Error::PageReferenceOutOfRange(17)) => true, _ => false });
        }
    }
}
<commit_msg>Fix warning; RawHeader should in fact be Copy<commit_after>// Copyright 2017 pdb Developers
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.

use common::*;
use source::*;

mod page_list;
use self::page_list::PageList;

use std::fmt;

type PageNumber = u32;

#[derive(Debug,Copy,Clone)]
struct Header {
    page_size: usize,
    maximum_valid_page_number: PageNumber,
}

impl Header {
    fn pages_needed_to_store(&self, bytes: usize) -> usize {
        (bytes + (self.page_size - 1)) / self.page_size
    }

    fn validate_page_number(&self, page_number: u32) -> Result<PageNumber> {
        if page_number == 0 || page_number > self.maximum_valid_page_number {
            Err(Error::PageReferenceOutOfRange(page_number))
        } else {
            Ok(page_number as PageNumber)
        }
    }
}

/// Represents a stream table at various stages of access
#[doc(hidden)]
#[derive(Debug)]
enum StreamTable<'s> {
    /// The MSF header gives us the size of the table in bytes, and the list of pages (usually one)
    /// where we can find the list of pages that contain the stream table.
    HeaderOnly { size_in_bytes: usize, stream_table_location_location: PageList },

    /// Given the HeaderOnly information, we can do an initial read to get the actual location of
    /// the stream table as a PageList.
    TableFound { stream_table_location: PageList },

    // Given the table location, we can access the stream table itself
    Available { stream_table_view: Box<SourceView<'s>> }
}

fn view<'s>(source: &mut Source<'s>, page_list: &PageList) -> Result<Box<SourceView<'s>>> {
    // view it
    let view = source.view(page_list.source_slices())?;

    // double check our Source
    // if the Source didn't return the requested bits, that's an implementation bug, so
    // assert instead of returning an error
    assert_eq!(view.as_slice().len(), page_list.len());

    // done
    Ok(view)
}

mod big {
    use super::*;

    pub const MAGIC: &'static [u8] = b"Microsoft C/C++ MSF 7.00\r\n\x1a\x44\x53\x00\x00\x00";

    /// The PDB header as stored on disk.
    /// See the Microsoft code for reference: https://github.com/Microsoft/microsoft-pdb/blob/082c5290e5aff028ae84e43affa8be717aa7af73/PDB/msf/msf.cpp#L946
    #[derive(Debug, Copy, Clone, Pread)]
    #[repr(C, packed)]
    struct RawHeader {
        magic: [u8; 32],
        page_size: u32,
        free_page_map: u32,
        pages_used: u32,
        directory_size: u32,
        _reserved: u32,
    }

    #[derive(Debug)]
    pub struct BigMSF<'s, S> {
        header: Header,
        source: S,
        stream_table: StreamTable<'s>,
    }

    impl<'s, S: Source<'s>> BigMSF<'s, S> {
        pub fn new(source: S, header_view: Box<SourceView>) -> Result<BigMSF<'s, S>> {
            let mut buf = ParseBuffer::from(header_view.as_slice());
            let header: RawHeader = buf.parse()?;

            if header.magic != MAGIC {
                return Err(Error::UnrecognizedFileFormat);
            }

            if header.page_size.count_ones() != 1 || header.page_size < 0x100
                || header.page_size > (128 * 0x10000) {
                    return Err(Error::InvalidPageSize(header.page_size));
                }

            let header_object = Header{
                page_size: header.page_size as usize,
                maximum_valid_page_number: header.pages_used,
            };

            // calculate how many pages are needed to store the stream table
            let size_of_stream_table_in_pages = header_object.pages_needed_to_store(header.directory_size as usize);

            // now: how many pages are needed to store the list of pages that store the stream table?
            // each page entry is a u32, so multiply by four
            let size_of_stream_table_page_list_in_pages = header_object.pages_needed_to_store(size_of_stream_table_in_pages * 4);

            // read the list of stream table page list pages, which immediately follow the header
            // yes, this is a stupid level of indirection
            let mut stream_table_page_list_page_list = PageList::new(header_object.page_size);
            for _ in 0..size_of_stream_table_page_list_in_pages {
                let n = buf.parse_u32()?;
                stream_table_page_list_page_list.push(header_object.validate_page_number(n)?);
            }

            // truncate the stream table location location to the correct size
            stream_table_page_list_page_list.truncate(size_of_stream_table_in_pages * 4);

            Ok(BigMSF{
                header: header_object,
                source: source,
                stream_table: StreamTable::HeaderOnly {
                    size_in_bytes: header.directory_size as usize,
                    stream_table_location_location: stream_table_page_list_page_list,
                },
            })
        }

        fn find_stream_table(&mut self) -> Result<()> {
            let mut new_stream_table: Option<StreamTable> = None;

            if let StreamTable::HeaderOnly { size_in_bytes, ref stream_table_location_location } = self.stream_table {
                // the header indicated we need to read size_in_pages page numbers from the
                // specified PageList.

                // ask to view the location location
                let location_location = view(&mut self.source, stream_table_location_location)?;

                // build a PageList
                let mut page_list = PageList::new(self.header.page_size);
                let mut buf = ParseBuffer::from(location_location.as_slice());
                while buf.len() > 0 {
                    let n = buf.parse_u32()?;
                    page_list.push(self.header.validate_page_number(n)?);
                }

                page_list.truncate(size_in_bytes);

                // remember what we learned
                new_stream_table = Some(StreamTable::TableFound {
                    stream_table_location: page_list,
                });
            }

            if let Some(st) = new_stream_table {
                self.stream_table = st;
            }

            Ok(())
        }

        fn make_stream_table_available(&mut self) -> Result<()> {
            // do the initial read if we must
            if let StreamTable::HeaderOnly { .. } = self.stream_table {
                self.find_stream_table()?;
            }

            // do we need to map the stream table itself?
            let mut new_stream_table: Option<StreamTable> = None;
            if let StreamTable::TableFound { ref stream_table_location } = self.stream_table {
                // yep
                // ask the source to view it
                let stream_table_view = view(&mut self.source, stream_table_location)?;

                // done
                new_stream_table = Some(StreamTable::Available {
                    stream_table_view: stream_table_view,
                });
            }

            if let Some(st) = new_stream_table {
                self.stream_table = st;
            }

            // stream table is available
            assert!(match &self.stream_table {
                &StreamTable::Available { .. } => true,
                _ => false
            });

            Ok(())
        }

        fn look_up_stream(&mut self, stream_number: u32) -> Result<PageList> {
            // ensure the stream table is available
            self.make_stream_table_available()?;

            let header = self.header;

            // declare the things we're going to find
            let bytes_in_stream: u32;
            let page_list: PageList;

            if let StreamTable::Available { ref stream_table_view } = self.stream_table {
                let stream_table_slice = stream_table_view.as_slice();
                let mut stream_table = ParseBuffer::from(stream_table_slice);

                // the stream table is structured as:
                // stream_count
                // 0..stream_count: size of stream in bytes (0xffffffff indicating "stream does not exist")
                // stream 0: PageNumber
                // stream 1: PageNumber, PageNumber
                // stream 2: PageNumber, PageNumber, PageNumber, PageNumber, PageNumber
                // stream 3: PageNumber, PageNumber, PageNumber, PageNumber
                // (number of pages determined by number of bytes)

                let stream_count = stream_table.parse_u32()?;

                // check if we've already outworn our welcome
                if stream_number >= stream_count {
                    return Err(Error::StreamNotFound(stream_number))
                }

                // we now have {stream_count} u32s describing the length of each stream

                // walk over the streams before the requested stream
                // we need to pay attention to how big each one is, since their page numbers come
                // before our page numbers in the stream table
                let mut page_numbers_to_skip: usize = 0;
                for _ in 0..stream_number {
                    let bytes = stream_table.parse_u32()?;
                    if bytes == 0xffffffff {
                        // stream is not present, ergo nothing to skip
                    } else {
                        page_numbers_to_skip += header.pages_needed_to_store(bytes as usize);
                    }
                }

                // read our stream's size
                bytes_in_stream = stream_table.parse_u32()?;
                if bytes_in_stream == 0xffffffff {
                    return Err(Error::StreamNotFound(stream_number))
                }
                let pages_in_stream = header.pages_needed_to_store(bytes_in_stream as usize);

                // skip the remaining streams' byte counts
                let _ = stream_table.take((stream_count - stream_number - 1) as usize * 4)?;

                // skip the preceding streams' page numbers
                let _ = stream_table.take((page_numbers_to_skip as usize) * 4)?;

                // we're now at the list of pages for our stream
                // accumulate them into a PageList
                let mut list = PageList::new(header.page_size);
                for _ in 0..pages_in_stream {
                    let page_number = stream_table.parse_u32()?;
                    list.push(self.header.validate_page_number(page_number)?);
                }

                // truncate to the size of the stream
                list.truncate(bytes_in_stream as usize);

                page_list = list;
            } else {
                unreachable!();
            }

            // done!
            Ok(page_list)
        }
    }

    impl<'s, S: Source<'s>> MSF<'s, S> for BigMSF<'s, S> {
        fn get(&mut self, stream_number: u32, limit: Option<usize>) -> Result<Stream<'s>> {
            // look up the stream
            let mut page_list = self.look_up_stream(stream_number)?;

            // apply any limits we have
            if let Some(limit) = limit {
                page_list.truncate(limit);
            }

            // now that we know where this stream lives, we can view it
            let view = view(&mut self.source, &page_list)?;

            // pack it into a Stream
            let stream = Stream {
                source_view: view,
            };

            Ok(stream)
        }
    }
}

mod small {
    pub const MAGIC: &'static [u8] = b"Microsoft C/C++ program database 2.00\r\n\x1a\x4a\x47";
    // TODO: implement SmallMSF
}

/// Represents a single Stream within the multi-stream file.
#[derive(Debug)]
pub struct Stream<'s> {
    source_view: Box<SourceView<'s>>,
}

impl<'s> Stream<'s> {
    #[inline]
    pub fn parse_buffer(&self) -> ParseBuffer {
        let slice = self.source_view.as_slice();
        ParseBuffer::from(slice)
    }
}

/// Provides access to a "multi-stream file", which is the container format used by PDBs.
pub trait MSF<'s, S> : fmt::Debug {
    /// Accesses a stream by stream number, optionally restricted by a byte limit.
    fn get(&mut self, stream_number: u32, limit: Option<usize>) -> Result<Stream<'s>>;
}

fn header_matches(actual: &[u8], expected: &[u8]) -> bool {
    actual.len() >= expected.len() && &actual[0..expected.len()] == expected
}

pub fn open_msf<'s, S: Source<'s> + 's>(mut source: S) -> Result<Box<MSF<'s, S> + 's>> {
    // map the header
    let mut header_location = PageList::new(4096);
    header_location.push(0);
    let header_view = view(&mut source, &header_location)?;

    // see if it's a BigMSF
    if header_matches(header_view.as_slice(), big::MAGIC) {
        // claimed!
        let bigmsf = big::BigMSF::new(source, header_view)?;
        return Ok(Box::new(bigmsf))
    }

    if header_matches(header_view.as_slice(), small::MAGIC) {
        // sorry
        return Err(Error::UnimplementedFeature("small MSF file format"));
    }

    Err(Error::UnrecognizedFileFormat)
}

#[cfg(test)]
mod tests {

    mod header {
        use msf::Header;
        use common::Error;

        #[test]
        fn test_pages_needed_to_store() {
            let h = Header{
                page_size: 4096,
                maximum_valid_page_number: 15,
            };
            assert_eq!(h.pages_needed_to_store(0), 0);
            assert_eq!(h.pages_needed_to_store(1), 1);
            assert_eq!(h.pages_needed_to_store(1024), 1);
            assert_eq!(h.pages_needed_to_store(2048), 1);
            assert_eq!(h.pages_needed_to_store(4095), 1);
            assert_eq!(h.pages_needed_to_store(4096), 1);
            assert_eq!(h.pages_needed_to_store(4097), 2);
        }

        #[test]
        fn test_validate_page_number() {
            let h = Header{
                page_size: 4096,
                maximum_valid_page_number: 15,
            };
            assert!(match h.validate_page_number(0) { Err(Error::PageReferenceOutOfRange(0)) => true, _ => false });
            assert!(match h.validate_page_number(1) { Ok(1) => true, _ => false });
            assert!(match h.validate_page_number(2) { Ok(2) => true, _ => false });
            assert!(match h.validate_page_number(14) { Ok(14) => true, _ => false });
            assert!(match h.validate_page_number(15) { Ok(15) => true, _ => false });
            assert!(match h.validate_page_number(16) { Err(Error::PageReferenceOutOfRange(16)) => true, _ => false });
            assert!(match h.validate_page_number(17) { Err(Error::PageReferenceOutOfRange(17)) => true, _ => false });
        }
    }
}
<|endoftext|>


<commit_before>use super::innovation::{Innovation, InnovationContainer};
use super::traits::{Distance, Genotype};
use super::alignment::Alignment;
use std::cmp;
use super::crossover::Crossover;
use rand::Rng;
use fixedbitset::FixedBitSet;
use std::collections::BTreeMap;

#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum NodeType {
    Input,
    Output,
    Hidden,
}

impl Default for NodeType {
    fn default() -> Self {
        NodeType::Hidden
    }
}

#[derive(Debug, Clone)]
pub struct NodeGene {
    pub node_type: NodeType,
}

// To avoid node collisions we use Innovation numbers instead of node
// ids.
#[derive(Debug, Clone)]
pub struct LinkGene {
    // This points to the NodeGene of that innovation
    pub source_node_gene: Innovation,
    // This points to the NodeGene of that innovation
    pub target_node_gene: Innovation,
    pub weight: f64,
    pub active: bool,
}

impl LinkGene {
    pub fn disable(&mut self) {
        assert!(self.active);
        self.active = false;
    }
}

#[derive(Debug, Clone)]
pub struct NetworkGenome {
    pub link_genes: InnovationContainer<LinkGene>,
    pub node_genes: InnovationContainer<NodeGene>,
}

#[derive(Debug, Clone)]
struct AdjMatrix {
    n: usize,
    m: FixedBitSet,
}

impl AdjMatrix {
    fn new(n: usize) -> AdjMatrix {
        AdjMatrix {
            n: n,
            m: FixedBitSet::with_capacity(n * n),
        }
    }

    fn set(&mut self, i: usize, j: usize) {
        self.m.insert(i * self.n + j);
    }

    fn contains(&self, i: usize, j: usize) -> bool {
        self.m.contains(i * self.n + j)
    }

    fn transitive_closure(&mut self) {
        loop {
            let mut counts = 0;
            for i in 0..self.n {
                for j in 0..self.n {
                    if self.contains(i, j) {
                        // look column for j
                        for k in 0..self.n {
                            if self.contains(j, k) {
                                if !self.contains(i, k) {
                                    self.set(i, k);
                                    counts += 1;
                                }
                            }
                        }
                    }
                }
            }
            if counts == 0 {
                break;
            }
        }
    }

    fn unconnected_pairs_no_cycle(&self) -> Vec<(usize, usize)> {
        let mut pairs = Vec::new();
        for i in 0..self.n {
            for j in 0..self.n {
                if i != j {
                    if !(self.contains(i, j) || self.contains(j, i)) {
                        // make sure we don't introduce a cycle
                        pairs.push((i, j));
                    }
                }
            }
        }
        pairs
    }
}

impl NetworkGenome {
    // Uses the crossover method C to recombine left and right.
    pub fn crossover<R: Rng, C: Crossover>(left: &Self, right: &Self, c: &C, rng: &mut R) -> Self {
        let new_link_genes = c.crossover(&left.link_genes, &right.link_genes, rng);
        let mut new_node_genes = c.crossover(&left.node_genes, &right.node_genes, rng);

        // Additionally make sure that all link genes have a correspondant node gene.
        // We do this also for disabled links!
        for link in new_link_genes.map.values() {
            new_node_genes.insert_from_either_or(link.source_node_gene,
                                                 &left.node_genes,
                                                 &right.node_genes);
            new_node_genes.insert_from_either_or(link.target_node_gene,
                                                 &left.node_genes,
                                                 &right.node_genes);
        }

        NetworkGenome {
            link_genes: new_link_genes,
            node_genes: new_node_genes,
        }
    }

    /// Constructs the adjacency matrix from the genome.
    fn adjacency_matrix(&self) -> (AdjMatrix, Vec<Option<Innovation>>) {
        // maps the node innovation to the index in the adjacency matrix
        let mut map_to_indices: BTreeMap<Innovation, usize> = BTreeMap::new();

        for &node_innov in self.node_genes.map.keys() {
            let next_idx = map_to_indices.len();
            let source_idx = *(map_to_indices.entry(node_innov).or_insert(next_idx));
        }

        for link in self.link_genes.map.values() {
            if link.active {
                let next_idx = map_to_indices.len();
                let source_idx = *(map_to_indices.entry(link.source_node_gene).or_insert(next_idx));
                let next_idx = map_to_indices.len();
                let target_idx = *(map_to_indices.entry(link.target_node_gene).or_insert(next_idx));
            }
        }
        let n = map_to_indices.len();
        // XXX: Use node_genes count to create the adj_matrix in one step.
        let mut adj_matrix = AdjMatrix::new(n);
        for link in self.link_genes.map.values() {
            if link.active {
                let source_idx = *map_to_indices.get(&link.source_node_gene).unwrap();
                let target_idx = *map_to_indices.get(&link.target_node_gene).unwrap();
                adj_matrix.set(source_idx, target_idx);
            }
        }
        // construct a reverse map
        let mut rev_map: Vec<Option<Innovation>> = (0..n).map(|_| None).collect();
        for (&k, &v) in map_to_indices.iter() {
            rev_map[v] = Some(k);
        }
        assert!(rev_map.iter().all(Option::is_some));

        (adj_matrix, rev_map)
    }

    /// Returns a random link gene innovation which is active. Or None if no such exists.
    pub fn find_random_active_link_gene<R: Rng>(&self, rng: &mut R) -> Option<Innovation> {
        let mut all_active_link_innovations = Vec::with_capacity(self.link_genes.len());
        for (&innov, link) in self.link_genes.map.iter() {
            if link.active {
                all_active_link_innovations.push(innov);
            }
        }
        rng.choose(&all_active_link_innovations).map(|&i| i)
    }

    /// Returns two node innvovations which are not yet connected and which does not
    /// create a cycle, or None if no such exists.
    pub fn find_unconnected_pair<R: Rng>(&self, rng: &mut R) -> Option<(Innovation, Innovation)> {
        // Construct binary adjacency matrix
        let (adj_matrix, rev_map) = self.adjacency_matrix();

        // generate the transitive closure.
        let transitive_closure = adj_matrix.clone().transitive_closure();

        // Construct an array of all currently unconnected nodes.
        let unconnected = adj_matrix.unconnected_pairs_no_cycle();

        // Out of all unconnected pairs, choose a random one.
        rng.choose(&unconnected).map(|&(src, target)| {
            // reverse map src and target to the node innovation
            let src_innovation = rev_map[src].unwrap();
            let target_innovation = rev_map[target].unwrap();

            (src_innovation, target_innovation)
        })
    }
}

impl Genotype for NetworkGenome {}

struct LinkGeneWeightDistance;

impl Distance<LinkGene> for LinkGeneWeightDistance {
    fn distance(&self, a: &LinkGene, b: &LinkGene) -> f64 {
        a.weight - b.weight
    }
}

pub struct LinkGeneListDistance {
    pub excess: f64,
    pub disjoint: f64,
    pub weight: f64,
}

impl Distance<InnovationContainer<LinkGene>> for LinkGeneListDistance {
    fn distance(&self,
                genes_left: &InnovationContainer<LinkGene>,
                genes_right: &InnovationContainer<LinkGene>)
                -> f64 {
        let max_len = cmp::max(genes_left.len(), genes_right.len());
        if max_len == 0 {
            return 0.0;
        }

        let mut matching = 0;
        let mut disjoint = 0;
        let mut excess = 0;
        let mut weight_dist = 0.0;

        genes_left.align(genes_right,
                         &mut |_, alignment| {
                             match alignment {
                                 Alignment::Match(gene_left, gene_right) => {
                                     matching += 1;
                                     weight_dist += LinkGeneWeightDistance.distance(gene_left,
                                                                                gene_right)
                                                                          .abs();
                                 }
                                 Alignment::DisjointLeft(_) | Alignment::DisjointRight(_) => {
                                     disjoint += 1;
                                 }
                                 Alignment::ExcessLeft(_) | Alignment::ExcessRight(_) => {
                                     excess += 1;
                                 }
                             }
                         });

        assert!(2 * matching + disjoint + excess == genes_left.len() + genes_right.len());

        self.excess * (excess as f64) / (max_len as f64) +
        self.disjoint * (disjoint as f64) / (max_len as f64) +
        self.weight *
        if matching > 0 {
            weight_dist / (matching as f64)
        } else {
            0.0
        }
    }
}

pub struct NetworkGenomeDistance {
    pub l: LinkGeneListDistance,
}

impl Distance<NetworkGenome> for NetworkGenomeDistance {
    fn distance(&self, genome_left: &NetworkGenome, genome_right: &NetworkGenome) -> f64 {
        self.l.distance(&genome_left.link_genes, &genome_right.link_genes)
    }
}
<commit_msg>Use the transitive closure to test for cycles.<commit_after>use super::innovation::{Innovation, InnovationContainer};
use super::traits::{Distance, Genotype};
use super::alignment::Alignment;
use std::cmp;
use super::crossover::Crossover;
use rand::Rng;
use fixedbitset::FixedBitSet;
use std::collections::BTreeMap;

#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum NodeType {
    Input,
    Output,
    Hidden,
}

impl Default for NodeType {
    fn default() -> Self {
        NodeType::Hidden
    }
}

#[derive(Debug, Clone)]
pub struct NodeGene {
    pub node_type: NodeType,
}

// To avoid node collisions we use Innovation numbers instead of node
// ids.
#[derive(Debug, Clone)]
pub struct LinkGene {
    // This points to the NodeGene of that innovation
    pub source_node_gene: Innovation,
    // This points to the NodeGene of that innovation
    pub target_node_gene: Innovation,
    pub weight: f64,
    pub active: bool,
}

impl LinkGene {
    pub fn disable(&mut self) {
        assert!(self.active);
        self.active = false;
    }
}

#[derive(Debug, Clone)]
pub struct NetworkGenome {
    pub link_genes: InnovationContainer<LinkGene>,
    pub node_genes: InnovationContainer<NodeGene>,
}

#[derive(Debug, Clone)]
struct AdjMatrix {
    n: usize,
    m: FixedBitSet,
}

impl AdjMatrix {
    fn new(n: usize) -> AdjMatrix {
        AdjMatrix {
            n: n,
            m: FixedBitSet::with_capacity(n * n),
        }
    }

    fn set(&mut self, i: usize, j: usize) {
        self.m.insert(i * self.n + j);
    }

    fn contains(&self, i: usize, j: usize) -> bool {
        self.m.contains(i * self.n + j)
    }

    /// This is O(n**4) in worst case.
    fn transitive_closure(mut self) -> AdjMatrix {
        loop {
            let mut counts = 0;
            for i in 0..self.n {
                for j in 0..self.n {
                    if self.contains(i, j) {
                        // look column for j
                        for k in 0..self.n {
                            if self.contains(j, k) {
                                if !self.contains(i, k) {
                                    self.set(i, k);
                                    counts += 1;
                                }
                            }
                        }
                    }
                }
            }
            if counts == 0 {
                break;
            }
        }
        self
    }

    fn unconnected_pairs_no_cycle(&self) -> Vec<(usize, usize)> {
        let mut pairs = Vec::new();
        for i in 0..self.n {
            for j in 0..self.n {
                if i != j {
                    if !(self.contains(i, j) || self.contains(j, i)) {
                        // make sure we don't introduce a cycle
                        pairs.push((i, j));
                    }
                }
            }
        }
        pairs
    }
}

impl NetworkGenome {
    // Uses the crossover method C to recombine left and right.
    pub fn crossover<R: Rng, C: Crossover>(left: &Self, right: &Self, c: &C, rng: &mut R) -> Self {
        let new_link_genes = c.crossover(&left.link_genes, &right.link_genes, rng);
        let mut new_node_genes = c.crossover(&left.node_genes, &right.node_genes, rng);

        // Additionally make sure that all link genes have a correspondant node gene.
        // We do this also for disabled links!
        for link in new_link_genes.map.values() {
            new_node_genes.insert_from_either_or(link.source_node_gene,
                                                 &left.node_genes,
                                                 &right.node_genes);
            new_node_genes.insert_from_either_or(link.target_node_gene,
                                                 &left.node_genes,
                                                 &right.node_genes);
        }

        NetworkGenome {
            link_genes: new_link_genes,
            node_genes: new_node_genes,
        }
    }

    /// Constructs the adjacency matrix from the genome.
    fn adjacency_matrix(&self) -> (AdjMatrix, Vec<Option<Innovation>>) {
        // maps the node innovation to the index in the adjacency matrix
        let mut map_to_indices: BTreeMap<Innovation, usize> = BTreeMap::new();

        for &node_innov in self.node_genes.map.keys() {
            let next_idx = map_to_indices.len();
            let source_idx = *(map_to_indices.entry(node_innov).or_insert(next_idx));
        }

        for link in self.link_genes.map.values() {
            if link.active {
                let next_idx = map_to_indices.len();
                let source_idx = *(map_to_indices.entry(link.source_node_gene).or_insert(next_idx));
                let next_idx = map_to_indices.len();
                let target_idx = *(map_to_indices.entry(link.target_node_gene).or_insert(next_idx));
            }
        }
        let n = map_to_indices.len();
        // XXX: Use node_genes count to create the adj_matrix in one step.
        let mut adj_matrix = AdjMatrix::new(n);
        for link in self.link_genes.map.values() {
            if link.active {
                let source_idx = *map_to_indices.get(&link.source_node_gene).unwrap();
                let target_idx = *map_to_indices.get(&link.target_node_gene).unwrap();
                adj_matrix.set(source_idx, target_idx);
            }
        }
        // construct a reverse map
        let mut rev_map: Vec<Option<Innovation>> = (0..n).map(|_| None).collect();
        for (&k, &v) in map_to_indices.iter() {
            rev_map[v] = Some(k);
        }
        assert!(rev_map.iter().all(Option::is_some));

        (adj_matrix, rev_map)
    }

    /// Returns a random link gene innovation which is active. Or None if no such exists.
    pub fn find_random_active_link_gene<R: Rng>(&self, rng: &mut R) -> Option<Innovation> {
        let mut all_active_link_innovations = Vec::with_capacity(self.link_genes.len());
        for (&innov, link) in self.link_genes.map.iter() {
            if link.active {
                all_active_link_innovations.push(innov);
            }
        }
        rng.choose(&all_active_link_innovations).map(|&i| i)
    }

    /// Returns two node innvovations which are not yet connected and which does not
    /// create a cycle. Return `None` if no such connection exists.
    pub fn find_unconnected_pair<R: Rng>(&self, rng: &mut R) -> Option<(Innovation, Innovation)> {
        // Construct binary adjacency matrix
        let (adj_matrix, rev_map) = self.adjacency_matrix();

        // generate the transitive closure.
        let transitive_closure = adj_matrix.transitive_closure();

        // Construct an array of all currently unconnected nodes.
        let unconnected = transitive_closure.unconnected_pairs_no_cycle();

        // Out of all unconnected pairs, choose a random one.
        rng.choose(&unconnected).map(|&(src, target)| {
            // reverse map src and target to the node innovation
            let src_innovation = rev_map[src].unwrap();
            let target_innovation = rev_map[target].unwrap();

            (src_innovation, target_innovation)
        })
    }
}

impl Genotype for NetworkGenome {}

struct LinkGeneWeightDistance;

impl Distance<LinkGene> for LinkGeneWeightDistance {
    fn distance(&self, a: &LinkGene, b: &LinkGene) -> f64 {
        a.weight - b.weight
    }
}

pub struct LinkGeneListDistance {
    pub excess: f64,
    pub disjoint: f64,
    pub weight: f64,
}

impl Distance<InnovationContainer<LinkGene>> for LinkGeneListDistance {
    fn distance(&self,
                genes_left: &InnovationContainer<LinkGene>,
                genes_right: &InnovationContainer<LinkGene>)
                -> f64 {
        let max_len = cmp::max(genes_left.len(), genes_right.len());
        if max_len == 0 {
            return 0.0;
        }

        let mut matching = 0;
        let mut disjoint = 0;
        let mut excess = 0;
        let mut weight_dist = 0.0;

        genes_left.align(genes_right,
                         &mut |_, alignment| {
                             match alignment {
                                 Alignment::Match(gene_left, gene_right) => {
                                     matching += 1;
                                     weight_dist += LinkGeneWeightDistance.distance(gene_left,
                                                                                gene_right)
                                                                          .abs();
                                 }
                                 Alignment::DisjointLeft(_) | Alignment::DisjointRight(_) => {
                                     disjoint += 1;
                                 }
                                 Alignment::ExcessLeft(_) | Alignment::ExcessRight(_) => {
                                     excess += 1;
                                 }
                             }
                         });

        assert!(2 * matching + disjoint + excess == genes_left.len() + genes_right.len());

        self.excess * (excess as f64) / (max_len as f64) +
        self.disjoint * (disjoint as f64) / (max_len as f64) +
        self.weight *
        if matching > 0 {
            weight_dist / (matching as f64)
        } else {
            0.0
        }
    }
}

pub struct NetworkGenomeDistance {
    pub l: LinkGeneListDistance,
}

impl Distance<NetworkGenome> for NetworkGenomeDistance {
    fn distance(&self, genome_left: &NetworkGenome, genome_right: &NetworkGenome) -> f64 {
        self.l.distance(&genome_left.link_genes, &genome_right.link_genes)
    }
}
<|endoftext|>


<commit_before>//! The neural network struct. See the module level documentation for usage.

use std::ops::Range;
use std::io;

use utils::Stack;
use file;
use gene::*;
use gene::GeneExtras::*;

const BIAS_GENE_VALUE: f64 = 1.0;

#[derive(Clone, Debug, PartialEq)]
pub struct Network {
    // size should be the length of the genome minus one, don't forget
    pub size: usize,
    pub genome: Vec<Gene>
}

impl Network {
    /// Evaluates the neural network with the given inputs, returning a vector of outputs. The encoding can
    /// encode recurrent connections and bias inputs, so an internal state is used. It is important to run
    /// the clear_state method before calling evaluate again, unless it is desired to allow data
    /// carry over from the previous evaluation, for example if the network is being used as a real
    /// time controller.
    ///
    /// If too little inputs are given, the empty inputs will have a value of zero. If too many
    /// inputs are given, the extras are discarded.
    ///
    /// # Examples
    ///
    /// ```no_run
    /// use cge::Network;
    ///
    /// // Load a neural network
    /// let mut network = Network::load_from_file("neural_network.ann").unwrap();
    ///
    /// // Get the output of the neural network the the specified inputs
    /// let result = network.evaluate(&vec![1.0, 1.0]);
    ///
    /// // Get the output of the neural network with no inputs
    /// let result = network.evaluate(&Vec::new());
    ///
    /// // Get the output of the neural network with too many inputs (extras aren't used)
    /// let result = network.evaluate(&vec![1.0, 1.0, 1.0]);
    ///
    /// // Let's say adder.ann is a file with a neural network with recurrent connections, used for
    /// // adding numbers together.
    /// let mut adder = Network::load_from_file("adder.ann").unwrap();
    ///
    /// // result_one will be 1.0
    /// let result_one = adder.evaluate(&vec![1.0]);
    ///
    /// // result_two will be 3.0
    /// let result_two = adder.evaluate(&vec![2.0]);
    ///
    /// // result_three will be 5.0
    /// let result_three = adder.evaluate(&vec![2.0]);
    ///
    /// // If this behavior is not desired, call the clear_state method between evaluations:
    /// let result_one = adder.evaluate(&vec![1.0]);
    ///
    /// adder.clear_state();
    ///
    /// // The 1.0 from the previous call is gone, so result_two will be 2.0
    /// let result_two = adder.evaluate(&vec![2.0]);
    /// ```
    pub fn evaluate(&mut self, inputs: &Vec<f64>) -> Vec<f64> {
        // Set inputs
        for gene in &mut self.genome {
            if let Input(ref mut current_value) = gene.variant {
                *current_value = match inputs.get(gene.id) {
                    Some(v) => *v,
                    None => 0.0
                }
            }
        }

        let size = self.size;
        self.evaluate_slice(0..size, true)
    }

    /// Clears the internal state of the neural network.
    pub fn clear_state(&mut self) {
        for gene in &mut self.genome {
            match gene.variant {
                Input(ref mut current_value) => {
                    *current_value = 0.0;
                },
                Neuron(ref mut current_value, _) => {
                    *current_value = 0.0;
                }
                _ => {}
            }
        }
    }

    /// Saves the neural network to a file. Returns an empty tuple on success, or an io error.
    ///
    /// # Examples
    ///
    /// ```no_run
    /// use cge::Network;
    ///
    /// // Create a neural network
    /// let network = Network {
    ///     size: 0,
    ///     genome: Vec::new()
    /// };
    ///
    /// // Save network to neural_network.ann
    /// network.save_to_file("neural_network.ann");
    /// ```
    pub fn save_to_file(&self, path: &str) -> io::Result<()> {
        file::write_network(self, path)
    }

    /// Loads a neural network from a file. No guarantees are made about the validity of the
    /// genome. Returns the network, or an io error. If the file is in a bad format,
    /// io::ErrorKind::InvalidData is returned.
    ///
    /// # Examples
    ///
    /// ```no_run
    /// use cge::Network;
    ///
    /// // Loads a network from the file neural_network.ann
    /// let mut network = Network::load_from_file("neural_network.ann").unwrap();
    /// ```
    pub fn load_from_file(path: &str) -> io::Result<Network> {
        file::read_network(path)
    }

    // Returns the output of sub-linear genome in the given range
    fn evaluate_slice(&mut self, range: Range<usize>, neuron_update: bool) -> Vec<f64> {
        let mut gene_index = range.end;
        // Initialize a stack for evaluating the neural network
        let mut stack = Stack::new();

        // Iterate backwards over the specified slice
        while gene_index >= range.start {
            let variant = self.genome[gene_index].variant.clone();

            // For debugging: uncomment these print statements
            // Perhaps add it as an option to help users
            // println!("{:?}", variant);

            match variant {
                Input(_) => {
                    // If the gene is an input, push its value multiplied by the inputs weight onto
                    // the stack
                    let (weight, _, value) = self.genome[gene_index].ref_input().unwrap();
                    stack.push(weight * value);
                },
                Neuron(_, _) => {
                    // If the gene is a neuron, pop a number (the neurons input count) of inputs
                    // off the stack, and push their sum multiplied by the neurons weight onto the
                    // stack
                    let (weight, _, value, inputs) = self.genome[gene_index].ref_mut_neuron().unwrap();
                    let new_value = stack.pop(*inputs).unwrap().iter().fold(0.0, |acc, i| acc + i);

                    if neuron_update {
                        *value = new_value;
                    }

                    stack.push(*weight * new_value);
                },
                Forward => {
                    // This is inefficient because it can run the neuron evaluation code multiple
                    // times
                    // TODO: Turn current value of neurons into a struct with a flag representing
                    // whether the neuron has been evaluated this network evaluation. Reset this
                    // flag after every network evaluation.

                    // If the gene is a forward jumper, evaluate the subnetwork starting at the
                    // neuron with id of the jumper, and push the result multiplied by the jumpers
                    // weight onto the stack
                    let weight = self.genome[gene_index].weight;
                    let id = self.genome[gene_index].id;
                    let subnetwork_range = self.get_subnetwork_index(id).unwrap();

                    let result = self.evaluate_slice(subnetwork_range, false);
                    // println!("{:?}", result);
                    stack.push(weight * result[0]);
                },
                Recurrent => {
                    // If the gene is a recurrent jumper, push the previous value of the neuron
                    // with the id of the jumper multiplied by the jumpers weight onto the stack
                    let gene = &self.genome[gene_index];
                    let neuron = &self.genome[self.get_neuron_index(gene.id).unwrap()];

                    if let Neuron(ref current_value, _) = neuron.variant {
                        stack.push(gene.weight * *current_value);
                    }
                },
                Bias => {
                    // If the gene is a bias input, push the bias constant multiplied by the genes
                    // weight onto the stack
                    let gene = &self.genome[gene_index];
                    stack.push(gene.weight * BIAS_GENE_VALUE);
                }
            }

            if gene_index == range.start {
                break;
            }

            gene_index -= 1;
            // println!("{:?}", stack.data);
        }

        stack.data
    }

    // Returns the start and end index of the subnetwork starting at the neuron with the given id,
    // or None if it does not exist
    fn get_subnetwork_index(&self, id: usize) -> Option<Range<usize>> {
        let start = match self.get_neuron_index(id) {
            Some(i) => i,
            None => return None
        };

        let mut end = start;
        let mut sum = 0;

        // Iterate through genes after the start index, modifying the sum each step
        // I could use an iterator here, but it would be messy
        for gene in &self.genome[start..self.size] {
            match gene.variant {
                Neuron(_, ref inputs) => {
                    sum += 1 - *inputs as i32;
                },
                _ => {
                    sum += 1;
                }
            }

            if sum == 1 {
                break;
            }

            end += 1;
        }

        if sum != 1 {
            None
        } else {
            Some(Range {
                start: start,
                end: end
            })
        }
    }

    // Returns the index of the neuron with the given id, or None if it does not exist
    fn get_neuron_index(&self, id: usize) -> Option<usize> {
        let mut result = None;

        for (i, gene) in self.genome.iter().enumerate() {
            if let Neuron(_, _) = gene.variant {
                if gene.id == id {
                    result = Some(i);
                }
            }
        }

        result
    }
}
<commit_msg>Reverted previous commit<commit_after>//! The neural network struct. See the module level documentation for usage.

use std::ops::Range;
use std::io;

use utils::Stack;
use file;
use gene::*;
use gene::GeneExtras::*;

const BIAS_GENE_VALUE: f64 = 1.0;

#[derive(Clone, Debug, PartialEq)]
pub struct Network {
    // size should be the length of the genome minus one, don't forget
    pub size: usize,
    pub genome: Vec<Gene>
}

impl Network {
    /// Evaluates the neural network with the given inputs, returning a vector of outputs. The encoding can
    /// encode recurrent connections and bias inputs, so an internal state is used. It is important to run
    /// the clear_state method before calling evaluate again, unless it is desired to allow data
    /// carry over from the previous evaluation, for example if the network is being used as a real
    /// time controller.
    ///
    /// If too little inputs are given, the empty inputs will have a value of zero. If too many
    /// inputs are given, the extras are discarded.
    ///
    /// # Examples
    ///
    /// ```no_run
    /// use cge::Network;
    ///
    /// // Load a neural network
    /// let mut network = Network::load_from_file("neural_network.ann").unwrap();
    ///
    /// // Get the output of the neural network the the specified inputs
    /// let result = network.evaluate(&vec![1.0, 1.0]);
    ///
    /// // Get the output of the neural network with no inputs
    /// let result = network.evaluate(&Vec::new());
    ///
    /// // Get the output of the neural network with too many inputs (extras aren't used)
    /// let result = network.evaluate(&vec![1.0, 1.0, 1.0]);
    ///
    /// // Let's say adder.ann is a file with a neural network with recurrent connections, used for
    /// // adding numbers together.
    /// let mut adder = Network::load_from_file("adder.ann").unwrap();
    ///
    /// // result_one will be 1.0
    /// let result_one = adder.evaluate(&vec![1.0]);
    ///
    /// // result_two will be 3.0
    /// let result_two = adder.evaluate(&vec![2.0]);
    ///
    /// // result_three will be 5.0
    /// let result_three = adder.evaluate(&vec![2.0]);
    ///
    /// // If this behavior is not desired, call the clear_state method between evaluations:
    /// let result_one = adder.evaluate(&vec![1.0]);
    ///
    /// adder.clear_state();
    ///
    /// // The 1.0 from the previous call is gone, so result_two will be 2.0
    /// let result_two = adder.evaluate(&vec![2.0]);
    /// ```
    pub fn evaluate(&mut self, inputs: &Vec<f64>) -> Vec<f64> {
        // Set inputs
        for gene in &mut self.genome {
            if let Input(ref mut current_value) = gene.variant {
                *current_value = match inputs.get(gene.id) {
                    Some(v) => *v,
                    None => 0.0
                }
            }
        }

        let size = self.size;
        self.evaluate_slice(0..size, true)
    }

    /// Clears the internal state of the neural network.
    pub fn clear_state(&mut self) {
        for gene in &mut self.genome {
            match gene.variant {
                Input(ref mut current_value) => {
                    *current_value = 0.0;
                },
                Neuron(ref mut current_value, _) => {
                    *current_value = 0.0;
                }
                _ => {}
            }
        }
    }

    /// Saves the neural network to a file. Returns an empty tuple on success, or an io error.
    ///
    /// # Examples
    ///
    /// ```no_run
    /// use cge::Network;
    ///
    /// // Create a neural network
    /// let network = Network {
    ///     size: 0,
    ///     genome: Vec::new()
    /// };
    ///
    /// // Save network to neural_network.ann
    /// network.save_to_file("neural_network.ann");
    /// ```
    pub fn save_to_file(&self, path: &str) -> io::Result<()> {
        file::write_network(self, path)
    }

    /// Loads a neural network from a file. No guarantees are made about the validity of the
    /// genome. Returns the network, or an io error. If the file is in a bad format,
    /// io::ErrorKind::InvalidData is returned.
    ///
    /// # Examples
    ///
    /// ```no_run
    /// use cge::Network;
    ///
    /// // Loads a network from the file neural_network.ann
    /// let mut network = Network::load_from_file("neural_network.ann").unwrap();
    /// ```
    pub fn load_from_file(path: &str) -> io::Result<Network> {
        file::read_network(path)
    }

    // Returns the output of sub-linear genome in the given range
    fn evaluate_slice(&mut self, range: Range<usize>, neuron_update: bool) -> Vec<f64> {
        let mut gene_index = range.end;
        // Initialize a stack for evaluating the neural network
        let mut stack = Stack::new();

        // Iterate backwards over the specified slice
        while gene_index >= range.start {
            let variant = self.genome[gene_index].variant.clone();

            // For debugging: uncomment these print statements
            // Perhaps add it as an option to help users
            // println!("{:?}", variant);

            match variant {
                Input(_) => {
                    // If the gene is an input, push its value multiplied by the inputs weight onto
                    // the stack
                    let (weight, _, value) = self.genome[gene_index].ref_input().unwrap();
                    stack.push(weight * value);
                },
                Neuron(_, _) => {
                    // If the gene is a neuron, pop a number (the neurons input count) of inputs
                    // off the stack, and push their sum multiplied by the neurons weight onto the
                    // stack
                    let (weight, _, value, inputs) = self.genome[gene_index].ref_mut_neuron().unwrap();
                    let new_value = stack.pop(*inputs).unwrap().iter().fold(0.0, |acc, i| acc + i);

                    if neuron_update {
                        *value = new_value;
                    }

                    stack.push(*weight * new_value);
                },
                Forward => {
                    // This is inefficient because it can run the neuron evaluation code multiple
                    // times
                    // TODO: Turn current value of neurons into a struct with a flag representing
                    // whether the neuron has been evaluated this network evaluation. Reset this
                    // flag after every network evaluation.

                    // If the gene is a forward jumper, evaluate the subnetwork starting at the
                    // neuron with id of the jumper, and push the result multiplied by the jumpers
                    // weight onto the stack
                    let weight = self.genome[gene_index].weight;
                    let id = self.genome[gene_index].id;
                    let subnetwork_range = self.get_subnetwork_index(id).unwrap();

                    let result = self.evaluate_slice(subnetwork_range, false);
                    // println!("{:?}", result);
                    stack.push(weight * result[0]);
                },
                Recurrent => {
                    // If the gene is a recurrent jumper, push the previous value of the neuron
                    // with the id of the jumper multiplied by the jumpers weight onto the stack
                    let gene = &self.genome[gene_index];
                    let neuron = &self.genome[self.get_neuron_index(gene.id).unwrap()];

                    if let Neuron(ref current_value, _) = neuron.variant {
                        stack.push(gene.weight * *current_value);
                    }
                },
                Bias => {
                    // If the gene is a bias input, push the bias constant multiplied by the genes
                    // weight onto the stack
                    let gene = &self.genome[gene_index];
                    stack.push(gene.weight * BIAS_GENE_VALUE);
                }
            }

            if gene_index == range.start {
                break;
            }

            gene_index -= 1;
            // println!("{:?}", stack.data);
        }

        stack.data
    }

    // Returns the start and end index of the subnetwork starting at the neuron with the given id,
    // or None if it does not exist
    fn get_subnetwork_index(&self, id: usize) -> Option<Range<usize>> {
        let start = match self.get_neuron_index(id) {
            Some(i) => i,
            None => return None
        };

        let mut end = start;
        let mut sum = 0;

        // Iterate through genes after the start index, modifying the sum each step
        // I could use an iterator here, but it would be messy
        for gene in &self.genome[start..self.size] {
            match gene.variant {
                Neuron(_, ref inputs) => {
                    sum += 1 - *inputs as i32;
                },
                _ => {
                    sum += 1;
                }
            }

            if sum == 1 {
                break;
            }

            end += 1;
        }

        if sum != 1 {
            None
        } else {
            Some(Range {
                start: start,
                end: end
            })
        }
    }

    /// Returns the index of the neuron with the given id, or None if it does not exist.
    pub fn get_neuron_index(&self, id: usize) -> Option<usize> {
        let mut result = None;

        for (i, gene) in self.genome.iter().enumerate() {
            if let Neuron(_, _) = gene.variant {
                if gene.id == id {
                    result = Some(i);
                }
            }
        }

        result
    }
}
<|endoftext|>


<commit_before>use crate::error::{Error, Result};
use std::ffi::OsStr;
use std::fmt::Debug;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::str;

pub struct Git<'a> {
    command: &'a str,
    cwd: &'a Path,
}

impl<'a> Git<'a> {
    pub fn command<S: AsRef<OsStr> + Debug>(&self, args: &[S]) -> Result<String> {
        let out = Command::new(&self.command)
            .arg("-C")
            .arg(self.cwd)
            .args(args)
            .output()?;
        if out.status.success() {
            Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
        } else {
            Err(Error::GitCommandError {
                stderr: String::from_utf8_lossy(&out.stderr)
                    .trim()
                    .replace('\n', " "),
                args: args.iter().map(|a| a.as_ref().to_owned()).collect(),
            })
        }
    }

    pub fn hash<S: AsRef<str>>(&self, commit: S) -> Result<String> {
        self.command(&["rev-parse", commit.as_ref()])
            .map_err(|err| Error::GitObjectNotFound {
                kind: "commit",
                object: commit.as_ref().to_string(),
                msg: format!("{}", err),
            })
    }

    pub fn tag_hash<S: AsRef<str>>(&self, tagname: S) -> Result<String> {
        let tagname = tagname.as_ref();
        let stdout = self
            .command(&["show-ref", "--tags", tagname])
            .map_err(|err| Error::GitObjectNotFound {
                kind: "tag name",
                object: tagname.to_string(),
                msg: format!("{}", err),
            })?;
        // Output must be in format '{rev} {ref name}'
        Ok(stdout.splitn(2, ' ').next().unwrap().to_string())
    }

    pub fn remote_url<S: AsRef<str>>(&self, name: S) -> Result<String> {
        // XXX:
        // `git remote get-url {name}` is not available because it's added recently (at 2.6.1).
        // Note that git installed in Ubuntu 14.04 is 1.9.1.
        let name = name.as_ref();
        self.command(&["config", "--get", &format!("remote.{}.url", name)])
            .map_err(|err| Error::GitObjectNotFound {
                kind: "remote",
                object: name.to_string(),
                msg: format!("{}", err),
            })
    }

    pub fn tracking_remote_url<S>(&self, branch: &Option<S>) -> Result<(String, String)>
    where
        S: AsRef<str>,
    {
        let rev = match branch {
            Some(b) => format!("{}@{{u}}", b.as_ref()),
            None => "@{u}".to_string(),
        };

        let out = match self.command(&["rev-parse", "--abbrev-ref", "--symbolic", rev.as_str()]) {
            Ok(stdout) => stdout,
            Err(Error::GitCommandError { ref stderr, .. })
                if stderr.contains("does not point to a branch") =>
            {
                return Ok((self.remote_url("origin")?, "origin".to_string()));
            }
            Err(err) => return Err(err),
        };

        // out is formatted as '{remote-name}/{branch-name}'
        match out.splitn(2, '/').next() {
            Some(u) => Ok((self.remote_url(u)?, u.to_string())),
            None => Err(Error::UnexpectedRemoteName(out.clone())),
        }
    }

    pub fn root_dir(&self) -> Result<PathBuf> {
        match self.command(&["rev-parse", "--show-toplevel"]) {
            Ok(stdout) => Ok(fs::canonicalize(stdout)?),
            Err(Error::GitCommandError { stderr, .. }) => Err(Error::GitRootDirNotFound {
                cwd: self.cwd.to_owned(),
                stderr: stderr,
            }),
            Err(e) => Err(e),
        }
    }

    pub fn current_branch(&self) -> Result<String> {
        self.command(&["rev-parse", "--abbrev-ref", "--symbolic", "HEAD"])
    }

    pub fn remote_contains<S, T>(&self, spec: S, remote_branch: T) -> Result<bool>
    where
        S: AsRef<str>,
        T: AsRef<str>,
    {
        self.command(&[
            "branch",
            "--remote",
            remote_branch.as_ref(),
            "--contains",
            spec.as_ref(),
        ])
        .map(|o| !o.is_empty())
    }

    // Returns {remote}/{branch}
    pub fn remote_branch<S: AsRef<str>, T: AsRef<str>>(
        &self,
        remote_name: &Option<S>,
        local_branch: &Option<T>,
    ) -> Result<String> {
        let remote = match remote_name {
            Some(r) => r.as_ref(),
            None => "{u}",
        };
        let branch = match local_branch {
            Some(b) => b.as_ref(),
            None => "",
        };
        let rev = format!("{}@{}", branch, remote);
        self.command(&["rev-parse", "--abbrev-ref", "--symbolic", rev.as_str()])
    }
}

impl<'a> Git<'a> {
    pub fn new(cwd: &'a Path, command: &'a str) -> Git<'a> {
        Git { command, cwd }
    }
}
<commit_msg>fix clippy error<commit_after>use crate::error::{Error, Result};
use std::ffi::OsStr;
use std::fmt::Debug;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::str;

pub struct Git<'a> {
    command: &'a str,
    cwd: &'a Path,
}

impl<'a> Git<'a> {
    pub fn command<S: AsRef<OsStr> + Debug>(&self, args: &[S]) -> Result<String> {
        let out = Command::new(&self.command)
            .arg("-C")
            .arg(self.cwd)
            .args(args)
            .output()?;
        if out.status.success() {
            Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
        } else {
            Err(Error::GitCommandError {
                stderr: String::from_utf8_lossy(&out.stderr)
                    .trim()
                    .replace('\n', " "),
                args: args.iter().map(|a| a.as_ref().to_owned()).collect(),
            })
        }
    }

    pub fn hash<S: AsRef<str>>(&self, commit: S) -> Result<String> {
        self.command(&["rev-parse", commit.as_ref()])
            .map_err(|err| Error::GitObjectNotFound {
                kind: "commit",
                object: commit.as_ref().to_string(),
                msg: format!("{}", err),
            })
    }

    pub fn tag_hash<S: AsRef<str>>(&self, tagname: S) -> Result<String> {
        let tagname = tagname.as_ref();
        let stdout = self
            .command(&["show-ref", "--tags", tagname])
            .map_err(|err| Error::GitObjectNotFound {
                kind: "tag name",
                object: tagname.to_string(),
                msg: format!("{}", err),
            })?;
        // Output must be in format '{rev} {ref name}'
        Ok(stdout.splitn(2, ' ').next().unwrap().to_string())
    }

    pub fn remote_url<S: AsRef<str>>(&self, name: S) -> Result<String> {
        // XXX:
        // `git remote get-url {name}` is not available because it's added recently (at 2.6.1).
        // Note that git installed in Ubuntu 14.04 is 1.9.1.
        let name = name.as_ref();
        self.command(&["config", "--get", &format!("remote.{}.url", name)])
            .map_err(|err| Error::GitObjectNotFound {
                kind: "remote",
                object: name.to_string(),
                msg: format!("{}", err),
            })
    }

    pub fn tracking_remote_url<S>(&self, branch: &Option<S>) -> Result<(String, String)>
    where
        S: AsRef<str>,
    {
        let rev = match branch {
            Some(b) => format!("{}@{{u}}", b.as_ref()),
            None => "@{u}".to_string(),
        };

        let out = match self.command(&["rev-parse", "--abbrev-ref", "--symbolic", rev.as_str()]) {
            Ok(stdout) => stdout,
            Err(Error::GitCommandError { ref stderr, .. })
                if stderr.contains("does not point to a branch") =>
            {
                return Ok((self.remote_url("origin")?, "origin".to_string()));
            }
            Err(err) => return Err(err),
        };

        // out is formatted as '{remote-name}/{branch-name}'
        match out.splitn(2, '/').next() {
            Some(u) => Ok((self.remote_url(u)?, u.to_string())),
            None => Err(Error::UnexpectedRemoteName(out.clone())),
        }
    }

    pub fn root_dir(&self) -> Result<PathBuf> {
        match self.command(&["rev-parse", "--show-toplevel"]) {
            Ok(stdout) => Ok(fs::canonicalize(stdout)?),
            Err(Error::GitCommandError { stderr, .. }) => Err(Error::GitRootDirNotFound {
                cwd: self.cwd.to_owned(),
                stderr,
            }),
            Err(e) => Err(e),
        }
    }

    pub fn current_branch(&self) -> Result<String> {
        self.command(&["rev-parse", "--abbrev-ref", "--symbolic", "HEAD"])
    }

    pub fn remote_contains<S, T>(&self, spec: S, remote_branch: T) -> Result<bool>
    where
        S: AsRef<str>,
        T: AsRef<str>,
    {
        self.command(&[
            "branch",
            "--remote",
            remote_branch.as_ref(),
            "--contains",
            spec.as_ref(),
        ])
        .map(|o| !o.is_empty())
    }

    // Returns {remote}/{branch}
    pub fn remote_branch<S: AsRef<str>, T: AsRef<str>>(
        &self,
        remote_name: &Option<S>,
        local_branch: &Option<T>,
    ) -> Result<String> {
        let remote = match remote_name {
            Some(r) => r.as_ref(),
            None => "{u}",
        };
        let branch = match local_branch {
            Some(b) => b.as_ref(),
            None => "",
        };
        let rev = format!("{}@{}", branch, remote);
        self.command(&["rev-parse", "--abbrev-ref", "--symbolic", rev.as_str()])
    }
}

impl<'a> Git<'a> {
    pub fn new(cwd: &'a Path, command: &'a str) -> Git<'a> {
        Git { command, cwd }
    }
}
<|endoftext|>


<commit_before>/*
 * UpdateHub
 * Copyright (C) 2017
 * O.S. Systems Sofware LTDA: contact@ossystems.com.br
 *
 * SPDX-License-Identifier:     GPL-2.0
 */

use serde_ini;

extern crate chrono;
use self::chrono::{DateTime, Utc};

use de_helpers::bool_from_str;

#[derive(Deserialize, PartialEq, Serialize)]
#[serde(rename_all = "PascalCase")]
pub struct PersistentSettings {
    pub polling: PersistentPolling,
    pub update: PersistentUpdate,
}

#[derive(Deserialize, PartialEq, Serialize)]
#[serde(rename_all = "PascalCase")]
pub struct PersistentPolling {
    #[serde(rename = "LastPoll")]
    pub last: DateTime<Utc>,
    #[serde(rename = "FirstPoll")]
    pub first: DateTime<Utc>,
    pub extra_interval: usize,
    pub retries: usize,
    #[serde(rename = "ProbeASAP")]
    #[serde(deserialize_with = "bool_from_str")]
    pub now: bool,
}

impl Default for PersistentPolling {
    fn default() -> Self {
        PersistentPolling {
            last: Utc::now(),
            first: Utc::now(),
            extra_interval: 0,
            retries: 0,
            now: false,
        }
    }
}

#[derive(Deserialize, PartialEq, Serialize)]
#[serde(rename_all = "PascalCase")]
pub struct PersistentUpdate {
    #[serde(rename = "UpgradeToInstallation")]
    pub upgrading_to: i8,
}

impl Default for PersistentUpdate {
    fn default() -> Self {
        PersistentUpdate { upgrading_to: -1 }
    }
}

#[cfg(test)]
mod ini_de {
    use super::*;

    #[test]
    fn ok() {
        let ini = r"
[Polling]
LastPoll=2017-01-01T00:00:00Z
FirstPoll=2017-02-02T00:00:00Z
ExtraInterval=4
Retries=5
ProbeASAP=false

[Update]
UpgradeToInstallation=1
";
        let expected = PersistentSettings {
            polling: PersistentPolling {
                last: "2017-01-01T00:00:00Z".parse::<DateTime<Utc>>().unwrap(),
                first: "2017-02-02T00:00:00Z".parse::<DateTime<Utc>>().unwrap(),
                extra_interval: 4,
                retries: 5,
                now: false,
            },
            update: PersistentUpdate { upgrading_to: 1 },
        };

        assert!(
            serde_ini::from_str::<PersistentSettings>(&ini)
                .map_err(|e| println!("{}", e))
                .ok() == Some(expected)
        );
    }
}
<commit_msg>persistent_settings: add ::new and derive(Default)<commit_after>/*
 * UpdateHub
 * Copyright (C) 2017
 * O.S. Systems Sofware LTDA: contact@ossystems.com.br
 *
 * SPDX-License-Identifier:     GPL-2.0
 */

use serde_ini;

extern crate chrono;
use self::chrono::{DateTime, Utc};

use de_helpers::bool_from_str;

#[derive(Default, Deserialize, PartialEq, Serialize)]
#[serde(rename_all = "PascalCase")]
pub struct PersistentSettings {
    pub polling: PersistentPolling,
    pub update: PersistentUpdate,
}

impl PersistentSettings {
    pub fn new() -> Self {
        PersistentSettings::default()
    }
}

#[derive(Deserialize, PartialEq, Serialize)]
#[serde(rename_all = "PascalCase")]
pub struct PersistentPolling {
    #[serde(rename = "LastPoll")]
    pub last: DateTime<Utc>,
    #[serde(rename = "FirstPoll")]
    pub first: DateTime<Utc>,
    pub extra_interval: usize,
    pub retries: usize,
    #[serde(rename = "ProbeASAP")]
    #[serde(deserialize_with = "bool_from_str")]
    pub now: bool,
}

impl Default for PersistentPolling {
    fn default() -> Self {
        PersistentPolling {
            last: Utc::now(),
            first: Utc::now(),
            extra_interval: 0,
            retries: 0,
            now: false,
        }
    }
}

#[derive(Deserialize, PartialEq, Serialize)]
#[serde(rename_all = "PascalCase")]
pub struct PersistentUpdate {
    #[serde(rename = "UpgradeToInstallation")]
    pub upgrading_to: i8,
}

impl Default for PersistentUpdate {
    fn default() -> Self {
        PersistentUpdate { upgrading_to: -1 }
    }
}

#[cfg(test)]
mod ini_de {
    use super::*;

    #[test]
    fn ok() {
        let ini = r"
[Polling]
LastPoll=2017-01-01T00:00:00Z
FirstPoll=2017-02-02T00:00:00Z
ExtraInterval=4
Retries=5
ProbeASAP=false

[Update]
UpgradeToInstallation=1
";
        let expected = PersistentSettings {
            polling: PersistentPolling {
                last: "2017-01-01T00:00:00Z".parse::<DateTime<Utc>>().unwrap(),
                first: "2017-02-02T00:00:00Z".parse::<DateTime<Utc>>().unwrap(),
                extra_interval: 4,
                retries: 5,
                now: false,
            },
            update: PersistentUpdate { upgrading_to: 1 },
        };

        assert!(
            serde_ini::from_str::<PersistentSettings>(&ini)
                .map_err(|e| println!("{}", e))
                .ok() == Some(expected)
        );
    }

    #[test]
    fn default() {
        let settings = PersistentSettings::new();
        let expected = PersistentSettings {
            polling: PersistentPolling {
                last: settings.polling.last,
                first: settings.polling.first,
                extra_interval: 0,
                retries: 0,
                now: false,
            },
            update: PersistentUpdate { upgrading_to: -1 },
        };

        assert!(Some(settings) == Some(expected));
    }
}
<|endoftext|>


<commit_before>pub extern crate rosc;

// Re-export rosc items.
//
// Remove `Osc` prefix as items are already namespaced via a module, e.g. `OscMessage` becomes
// `osc::Message`.
#[doc(inline)]
pub use self::rosc::{decoder, encoder, OscBundle as Bundle, OscColor as Color,
                     OscMessage as Message, OscMidiMessage as MidiMessage, OscError as Error,
                     OscType as Type};
pub use self::recv::Receiver;
pub use self::send::Sender;

use std;
use std::net::{Ipv4Addr, SocketAddr};

pub mod recv;
pub mod send;

/// Indicates that a `Sender` is not currently connected to a target address, and that the target
/// address will have to be supplied manually when sending packets.
pub struct Unconnected;

/// Indicates that a `Sender` is currently connected to a specific target address.
pub struct Connected {
    addr: SocketAddr,
}

/// An *OSC packet* can contain an OSC message or a bundle of nested packets which is called an
/// *OSC bundle*.
#[derive(Clone, Debug, PartialEq)]
pub enum Packet {
    Message(Message),
    Bundle(Bundle),
}

impl From<Message> for Packet {
    fn from(msg: Message) -> Self {
        Packet::Message(msg)
    }
}

impl From<Bundle> for Packet {
    fn from(bundle: Bundle) -> Self {
        Packet::Bundle(bundle)
    }
}

impl From<rosc::OscPacket> for Packet {
    fn from(packet: rosc::OscPacket) -> Self {
        match packet {
            rosc::OscPacket::Message(msg) => msg.into(),
            rosc::OscPacket::Bundle(bundle) => bundle.into(),
        }
    }
}

impl<A> From<(A, Vec<Type>)> for Packet
    where A: Into<String>,
{
    fn from((addr, args): (A, Vec<Type>)) -> Self {
        msg(addr, args).into()
    }
}

impl Into<rosc::OscPacket> for Packet {
    fn into(self) -> rosc::OscPacket {
        match self {
            Packet::Message(msg) => rosc::OscPacket::Message(msg),
            Packet::Bundle(bundle) => rosc::OscPacket::Bundle(bundle),
        }
    }
}

impl Packet {
    /// A recursive function that unfolds the packet into the end of the given buffer of messages.
    pub fn unfold(self, msgs: &mut Vec<Message>) {
        match self {
            Packet::Message(msg) => msgs.push(msg),
            Packet::Bundle(bundle) => for packet in bundle.content {
                Packet::unfold(packet.into(), msgs);
            },
        }
    }

    /// Convert the `Packet` into a Vec containing all `Message`s contained within.
    ///
    /// This uses the `unfold` method internally.
    pub fn into_msgs(self) -> Vec<Message> {
        let mut msgs = vec![];
        self.unfold(&mut msgs);
        msgs
    }
}

/// The default local IP address.
pub fn default_ipv4_addr() -> Ipv4Addr {
    Ipv4Addr::new(127, 0, 0, 1)
}

/// A simple wrapper around the most commonly used `Receiver` constructor.
pub fn receiver(port: u16) -> Result<Receiver, std::io::Error> {
    Receiver::bind(port)
}

/// A simple wrapper around the most commonly used `Sender` constructor.
pub fn sender() -> Result<Sender, std::io::Error> {
    Sender::bind()
}

/// A simplified constructor for an OSC `Message`.
pub fn msg<A>(addr: A, args: Vec<Type>) -> Message
    where A: Into<String>,
{
    let addr = addr.into();
    let args = Some(args);
    Message { addr, args }
}

/// Decodes the given slice of `bytes` into a `Packet`.
///
/// Returns an `Error` if the slice does not contain a valid OSC packet.
pub fn decode(bytes: &[u8]) -> Result<Packet, Error> {
    rosc::decoder::decode(bytes).map(|p| p.into())
}

/// Encodes the given `Packet` into a `Vec` of bytes.
///
/// Returns an `Error` if the packet is invalid.
pub fn encode(packet: Packet) -> Result<Vec<u8>, Error> {
    let rosc_packet = packet.into();
    rosc::encoder::encode(&rosc_packet)
}

/// Errors that might occur whilst attempting to send or receive an OSC packet.
#[derive(Debug)]
pub enum CommunicationError {
    Io(std::io::Error),
    Osc(Error),
    Poisoned,
}

impl From<std::io::Error> for CommunicationError {
    fn from(err: std::io::Error) -> Self {
        CommunicationError::Io(err)
    }
}

impl From<Error> for CommunicationError {
    fn from(err: Error) -> Self {
        CommunicationError::Osc(err)
    }
}

impl<T> From<std::sync::PoisonError<T>> for CommunicationError {
    fn from(_: std::sync::PoisonError<T>) -> Self {
        CommunicationError::Poisoned
    }
}

impl std::error::Error for CommunicationError {
    fn description(&self) -> &str {
        match *self {
            CommunicationError::Io(ref err) => std::error::Error::description(err),
            // TODO: Error isn't implemented for OscError - should fix this upstream.
            CommunicationError::Osc(ref _err) => "Failed to decode the OSC packet",
            // CommunicationError::Osc(ref err) => std::error::Error::description(err),
            CommunicationError::Poisoned => "The inner buffer's mutex was poisoned",
        }
    }
    fn cause(&self) -> Option<&std::error::Error> {
        match *self {
            CommunicationError::Io(ref err) => Some(err),
            // TODO: Error isn't implemented for OscError - should fix this upstream.
            CommunicationError::Osc(ref _err) => None,
            // CommunicationError::Osc(ref err) => Some(err),
            _ => None,
        }
    }
}

impl std::fmt::Display for CommunicationError {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(f, "{}", std::error::Error::description(self))
    }
}
<commit_msg>Change OSC default binding address from `127.0.0.1` to `0.0.0.0`<commit_after>pub extern crate rosc;

// Re-export rosc items.
//
// Remove `Osc` prefix as items are already namespaced via a module, e.g. `OscMessage` becomes
// `osc::Message`.
#[doc(inline)]
pub use self::rosc::{decoder, encoder, OscBundle as Bundle, OscColor as Color,
                     OscMessage as Message, OscMidiMessage as MidiMessage, OscError as Error,
                     OscType as Type};
pub use self::recv::Receiver;
pub use self::send::Sender;

use std;
use std::net::{Ipv4Addr, SocketAddr};

pub mod recv;
pub mod send;

/// Indicates that a `Sender` is not currently connected to a target address, and that the target
/// address will have to be supplied manually when sending packets.
pub struct Unconnected;

/// Indicates that a `Sender` is currently connected to a specific target address.
pub struct Connected {
    addr: SocketAddr,
}

/// An *OSC packet* can contain an OSC message or a bundle of nested packets which is called an
/// *OSC bundle*.
#[derive(Clone, Debug, PartialEq)]
pub enum Packet {
    Message(Message),
    Bundle(Bundle),
}

impl From<Message> for Packet {
    fn from(msg: Message) -> Self {
        Packet::Message(msg)
    }
}

impl From<Bundle> for Packet {
    fn from(bundle: Bundle) -> Self {
        Packet::Bundle(bundle)
    }
}

impl From<rosc::OscPacket> for Packet {
    fn from(packet: rosc::OscPacket) -> Self {
        match packet {
            rosc::OscPacket::Message(msg) => msg.into(),
            rosc::OscPacket::Bundle(bundle) => bundle.into(),
        }
    }
}

impl<A> From<(A, Vec<Type>)> for Packet
    where A: Into<String>,
{
    fn from((addr, args): (A, Vec<Type>)) -> Self {
        msg(addr, args).into()
    }
}

impl Into<rosc::OscPacket> for Packet {
    fn into(self) -> rosc::OscPacket {
        match self {
            Packet::Message(msg) => rosc::OscPacket::Message(msg),
            Packet::Bundle(bundle) => rosc::OscPacket::Bundle(bundle),
        }
    }
}

impl Packet {
    /// A recursive function that unfolds the packet into the end of the given buffer of messages.
    pub fn unfold(self, msgs: &mut Vec<Message>) {
        match self {
            Packet::Message(msg) => msgs.push(msg),
            Packet::Bundle(bundle) => for packet in bundle.content {
                Packet::unfold(packet.into(), msgs);
            },
        }
    }

    /// Convert the `Packet` into a Vec containing all `Message`s contained within.
    ///
    /// This uses the `unfold` method internally.
    pub fn into_msgs(self) -> Vec<Message> {
        let mut msgs = vec![];
        self.unfold(&mut msgs);
        msgs
    }
}

/// The default local IP address.
pub fn default_ipv4_addr() -> Ipv4Addr {
    Ipv4Addr::new(0, 0, 0, 0)
}

/// A simple wrapper around the most commonly used `Receiver` constructor.
pub fn receiver(port: u16) -> Result<Receiver, std::io::Error> {
    Receiver::bind(port)
}

/// A simple wrapper around the most commonly used `Sender` constructor.
pub fn sender() -> Result<Sender, std::io::Error> {
    Sender::bind()
}

/// A simplified constructor for an OSC `Message`.
pub fn msg<A>(addr: A, args: Vec<Type>) -> Message
    where A: Into<String>,
{
    let addr = addr.into();
    let args = Some(args);
    Message { addr, args }
}

/// Decodes the given slice of `bytes` into a `Packet`.
///
/// Returns an `Error` if the slice does not contain a valid OSC packet.
pub fn decode(bytes: &[u8]) -> Result<Packet, Error> {
    rosc::decoder::decode(bytes).map(|p| p.into())
}

/// Encodes the given `Packet` into a `Vec` of bytes.
///
/// Returns an `Error` if the packet is invalid.
pub fn encode(packet: Packet) -> Result<Vec<u8>, Error> {
    let rosc_packet = packet.into();
    rosc::encoder::encode(&rosc_packet)
}

/// Errors that might occur whilst attempting to send or receive an OSC packet.
#[derive(Debug)]
pub enum CommunicationError {
    Io(std::io::Error),
    Osc(Error),
    Poisoned,
}

impl From<std::io::Error> for CommunicationError {
    fn from(err: std::io::Error) -> Self {
        CommunicationError::Io(err)
    }
}

impl From<Error> for CommunicationError {
    fn from(err: Error) -> Self {
        CommunicationError::Osc(err)
    }
}

impl<T> From<std::sync::PoisonError<T>> for CommunicationError {
    fn from(_: std::sync::PoisonError<T>) -> Self {
        CommunicationError::Poisoned
    }
}

impl std::error::Error for CommunicationError {
    fn description(&self) -> &str {
        match *self {
            CommunicationError::Io(ref err) => std::error::Error::description(err),
            // TODO: Error isn't implemented for OscError - should fix this upstream.
            CommunicationError::Osc(ref _err) => "Failed to decode the OSC packet",
            // CommunicationError::Osc(ref err) => std::error::Error::description(err),
            CommunicationError::Poisoned => "The inner buffer's mutex was poisoned",
        }
    }
    fn cause(&self) -> Option<&std::error::Error> {
        match *self {
            CommunicationError::Io(ref err) => Some(err),
            // TODO: Error isn't implemented for OscError - should fix this upstream.
            CommunicationError::Osc(ref _err) => None,
            // CommunicationError::Osc(ref err) => Some(err),
            _ => None,
        }
    }
}

impl std::fmt::Display for CommunicationError {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(f, "{}", std::error::Error::description(self))
    }
}
<|endoftext|>


<commit_before>mod analysis_stats;

use std::{fs, io::Read, path::Path};

use clap::{App, Arg, SubCommand};
use ra_ide_api::file_structure;
use ra_syntax::{SourceFile, TreeArc, AstNode};
use tools::collect_tests;
use flexi_logger::Logger;
use ra_prof::profile;

type Result<T> = ::std::result::Result<T, failure::Error>;

fn main() -> Result<()> {
    Logger::with_env().start()?;
    let matches = App::new("ra-cli")
        .setting(clap::AppSettings::SubcommandRequiredElseHelp)
        .subcommand(
            SubCommand::with_name("render-test")
                .arg(Arg::with_name("line").long("--line").required(true).takes_value(true))
                .arg(Arg::with_name("file").long("--file").required(true).takes_value(true)),
        )
        .subcommand(SubCommand::with_name("parse").arg(Arg::with_name("no-dump").long("--no-dump")))
        .subcommand(SubCommand::with_name("symbols"))
        .subcommand(
            SubCommand::with_name("analysis-stats")
                .arg(Arg::with_name("verbose").short("v"))
                .arg(Arg::with_name("only").short("o").takes_value(true))
                .arg(Arg::with_name("path")),
        )
        .get_matches();
    match matches.subcommand() {
        ("parse", Some(matches)) => {
            let _p = profile("parsing");
            let file = file()?;
            if !matches.is_present("no-dump") {
                println!("{}", file.syntax().debug_dump());
            }
            ::std::mem::forget(file);
        }
        ("symbols", _) => {
            let file = file()?;
            for s in file_structure(&file) {
                println!("{:?}", s);
            }
        }
        ("render-test", Some(matches)) => {
            let file = matches.value_of("file").unwrap();
            let file = Path::new(file);
            let line: usize = matches.value_of("line").unwrap().parse()?;
            let line = line - 1;
            let (test, tree) = render_test(file, line)?;
            println!("{}\n{}", test, tree);
        }
        ("analysis-stats", Some(matches)) => {
            let verbose = matches.is_present("verbose");
            let path = matches.value_of("path").unwrap_or("");
            let only = matches.value_of("only");
            analysis_stats::run(verbose, path, only)?;
        }
        _ => unreachable!(),
    }
    Ok(())
}

fn file() -> Result<TreeArc<SourceFile>> {
    let text = read_stdin()?;
    Ok(SourceFile::parse(&text))
}

fn read_stdin() -> Result<String> {
    let mut buff = String::new();
    ::std::io::stdin().read_to_string(&mut buff)?;
    Ok(buff)
}

fn render_test(file: &Path, line: usize) -> Result<(String, String)> {
    let text = fs::read_to_string(file)?;
    let tests = collect_tests(&text);
    let test = tests.into_iter().find(|(start_line, t)| {
        *start_line <= line && line <= *start_line + t.text.lines().count()
    });
    let test = match test {
        None => failure::bail!("No test found at line {} at {}", line, file.display()),
        Some((_start_line, test)) => test,
    };
    let file = SourceFile::parse(&test.text);
    let tree = file.syntax().debug_dump();
    Ok((test.text, tree))
}
<commit_msg>drop obsolete render test subcommand<commit_after>mod analysis_stats;

use std::io::Read;

use clap::{App, Arg, SubCommand};
use ra_ide_api::file_structure;
use ra_syntax::{SourceFile, TreeArc, AstNode};
use flexi_logger::Logger;
use ra_prof::profile;

type Result<T> = ::std::result::Result<T, failure::Error>;

fn main() -> Result<()> {
    Logger::with_env().start()?;
    let matches = App::new("ra-cli")
        .setting(clap::AppSettings::SubcommandRequiredElseHelp)
        .subcommand(SubCommand::with_name("parse").arg(Arg::with_name("no-dump").long("--no-dump")))
        .subcommand(SubCommand::with_name("symbols"))
        .subcommand(
            SubCommand::with_name("analysis-stats")
                .arg(Arg::with_name("verbose").short("v"))
                .arg(Arg::with_name("only").short("o").takes_value(true))
                .arg(Arg::with_name("path")),
        )
        .get_matches();
    match matches.subcommand() {
        ("parse", Some(matches)) => {
            let _p = profile("parsing");
            let file = file()?;
            if !matches.is_present("no-dump") {
                println!("{}", file.syntax().debug_dump());
            }
            ::std::mem::forget(file);
        }
        ("symbols", _) => {
            let file = file()?;
            for s in file_structure(&file) {
                println!("{:?}", s);
            }
        }
        ("analysis-stats", Some(matches)) => {
            let verbose = matches.is_present("verbose");
            let path = matches.value_of("path").unwrap_or("");
            let only = matches.value_of("only");
            analysis_stats::run(verbose, path, only)?;
        }
        _ => unreachable!(),
    }
    Ok(())
}

fn file() -> Result<TreeArc<SourceFile>> {
    let text = read_stdin()?;
    Ok(SourceFile::parse(&text))
}

fn read_stdin() -> Result<String> {
    let mut buff = String::new();
    ::std::io::stdin().read_to_string(&mut buff)?;
    Ok(buff)
}
<|endoftext|>