<commit_before>use LalafellBot;
use listeners::{ListenerManager, CommandListener};
use commands::*;

use xivdb::error::*;

use std::sync::Arc;

pub fn listeners(bot: Arc<LalafellBot>) -> Result<()> {
  let mut listeners = bot.listeners.lock().unwrap();
  listeners.push(box command_listener(bot.clone()));
  for listener in &bot.config.listeners {
    let listener = ListenerManager::from_config(bot.clone(), listener).chain_err(|| format!("could not create listener {}", listener.name))?;
    listeners.push(listener);
  }
  Ok(())
}

fn command_listener<'a>(bot: Arc<LalafellBot>) -> CommandListener<'a> {
  let mut command_listener = CommandListener::new(bot.clone());
  command_listener.add_command(&["race"], box RaceCommand::new(bot.clone()));
  command_listener.add_command(&["tag"], box TagCommand::new(bot.clone()));
  command_listener.add_command(&["autotag"], box AutoTagCommand::new(bot.clone()));
  command_listener.add_command(&["viewtag"], box ViewTagCommand::new(bot.clone()));
  command_listener.add_command(&["updatetags"], box UpdateTagsCommand::new(bot.clone()));
  command_listener.add_command(&["savedatabase"], box SaveDatabaseCommand::new(bot.clone()));
  command_listener.add_command(&["verify"], box VerifyCommand::new(bot.clone()));
  command_listener.add_command(&["referencecount"], box ReferenceCountCommand::new(bot.clone()));
  command_listener.add_command(&["join"], box JoinCommand::new(bot.clone()));
  command_listener.add_command(&["leave"], box LeaveCommand::new(bot.clone()));
  command_listener.add_command(&["play"], box PlayCommand::new(bot.clone()));
  command_listener
}
<commit_msg>fix: disable voice commands<commit_after>use LalafellBot;
use listeners::{ListenerManager, CommandListener};
use commands::*;

use xivdb::error::*;

use std::sync::Arc;

pub fn listeners(bot: Arc<LalafellBot>) -> Result<()> {
  let mut listeners = bot.listeners.lock().unwrap();
  listeners.push(box command_listener(bot.clone()));
  for listener in &bot.config.listeners {
    let listener = ListenerManager::from_config(bot.clone(), listener).chain_err(|| format!("could not create listener {}", listener.name))?;
    listeners.push(listener);
  }
  Ok(())
}

fn command_listener<'a>(bot: Arc<LalafellBot>) -> CommandListener<'a> {
  let mut command_listener = CommandListener::new(bot.clone());
  command_listener.add_command(&["race"], box RaceCommand::new(bot.clone()));
  command_listener.add_command(&["tag"], box TagCommand::new(bot.clone()));
  command_listener.add_command(&["autotag"], box AutoTagCommand::new(bot.clone()));
  command_listener.add_command(&["viewtag"], box ViewTagCommand::new(bot.clone()));
  command_listener.add_command(&["updatetags"], box UpdateTagsCommand::new(bot.clone()));
  command_listener.add_command(&["savedatabase"], box SaveDatabaseCommand::new(bot.clone()));
  command_listener.add_command(&["verify"], box VerifyCommand::new(bot.clone()));
  command_listener.add_command(&["referencecount"], box ReferenceCountCommand::new(bot.clone()));
  // command_listener.add_command(&["join"], box JoinCommand::new(bot.clone()));
  // command_listener.add_command(&["leave"], box LeaveCommand::new(bot.clone()));
  // command_listener.add_command(&["play"], box PlayCommand::new(bot.clone()));
  command_listener
}
<|endoftext|>


<commit_before>// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

//! Defines functionality for creating guest memory snapshots.

// Currently only used on x86_64.
#![cfg(target_arch = "x86_64")]

use std::fmt::{Display, Formatter};
use std::fs::File;
use std::io::SeekFrom;

use versionize::{VersionMap, Versionize, VersionizeResult};
use versionize_derive::Versionize;
use vm_memory::{
    Bytes, FileOffset, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryMmap,
    GuestMemoryRegion, GuestRegionMmap, MemoryRegionAddress, MmapRegion,
};

use crate::DirtyBitmap;

/// State of a guest memory region saved to file/buffer.
#[derive(Debug, PartialEq, Versionize)]
pub struct GuestMemoryRegionState {
    /// Base address.
    pub base_address: u64,
    /// Region size.
    pub size: usize,
    /// Offset in file/buffer where the region is saved.
    pub offset: u64,
}

/// Guest memory state.
#[derive(Debug, Default, PartialEq, Versionize)]
pub struct GuestMemoryState {
    /// List of regions.
    pub regions: Vec<GuestMemoryRegionState>,
}

/// Defines the interface for snapshotting memory.
pub trait SnapshotMemory
where
    Self: Sized,
{
    /// Describes GuestMemoryMmap through a GuestMemoryState struct.
    fn describe(&self) -> GuestMemoryState;
    /// Dumps all contents of GuestMemoryMmap to a writer.
    fn dump<T: std::io::Write>(&self, writer: &mut T) -> std::result::Result<(), Error>;
    /// Dumps all pages of GuestMemoryMmap present in `dirty_bitmap` to a writer.
    fn dump_dirty<T: std::io::Write + std::io::Seek>(
        &self,
        writer: &mut T,
        dirty_bitmap: &DirtyBitmap,
    ) -> std::result::Result<(), Error>;
    /// Creates a GuestMemoryMmap given a `file` containing the data
    /// and a `state` containing mapping information.
    fn restore(
        file: &File,
        state: &GuestMemoryState,
        track_dirty_pages: bool,
    ) -> std::result::Result<Self, Error>;
}

/// Errors associated with dumping guest memory to file.
#[derive(Debug)]
pub enum Error {
    /// Cannot access file.
    FileHandle(std::io::Error),
    /// Cannot create memory.
    CreateMemory(vm_memory::Error),
    /// Cannot create region.
    CreateRegion(vm_memory::mmap::MmapRegionError),
    /// Cannot dump memory.
    WriteMemory(GuestMemoryError),
}

impl Display for Error {
    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
        use self::Error::*;
        match self {
            FileHandle(err) => write!(f, "Cannot access file: {:?}", err),
            CreateMemory(err) => write!(f, "Cannot create memory: {:?}", err),
            CreateRegion(err) => write!(f, "Cannot create memory region: {:?}", err),
            WriteMemory(err) => write!(f, "Cannot dump memory: {:?}", err),
        }
    }
}

impl SnapshotMemory for GuestMemoryMmap {
    /// Describes GuestMemoryMmap through a GuestMemoryState struct.
    fn describe(&self) -> GuestMemoryState {
        let mut guest_memory_state = GuestMemoryState::default();
        let mut offset = 0;
        let _: std::result::Result<(), ()> = self.with_regions_mut(|_, region| {
            guest_memory_state.regions.push(GuestMemoryRegionState {
                base_address: region.start_addr().0,
                size: region.len() as usize,
                offset,
            });

            offset += region.len();
            Ok(())
        });
        guest_memory_state
    }

    /// Dumps all contents of GuestMemoryMmap to a writer.
    fn dump<T: std::io::Write>(&self, writer: &mut T) -> std::result::Result<(), Error> {
        self.with_regions_mut(|_, region| {
            region.write_all_to(MemoryRegionAddress(0), writer, region.len() as usize)
        })
        .map_err(Error::WriteMemory)
    }

    /// Dumps all pages of GuestMemoryMmap present in `dirty_bitmap` to a writer.
    fn dump_dirty<T: std::io::Write + std::io::Seek>(
        &self,
        writer: &mut T,
        dirty_bitmap: &DirtyBitmap,
    ) -> std::result::Result<(), Error> {
        let page_size = sysconf::page::pagesize();
        let mut writer_offset = 0;

        self.with_regions_mut(|slot, region| {
            let bitmap = dirty_bitmap.get(&slot).unwrap();
            let mut write_size = 0;
            let mut dirty_batch_start: u64 = 0;

            for (i, v) in bitmap.iter().enumerate() {
                for j in 0..64 {
                    let is_dirty_page = ((v >> j) & 1u64) != 0u64;
                    if is_dirty_page {
                        let page_offset = ((i * 64) + j) * page_size;
                        // We are at the start of a new batch of dirty pages.
                        if write_size == 0 {
                            // Seek forward over the unmodified pages.
                            writer
                                .seek(SeekFrom::Start(writer_offset + page_offset as u64))
                                .unwrap();
                            dirty_batch_start = page_offset as u64;
                        }
                        write_size += page_size;
                    } else if write_size > 0 {
                        // We are at the end of a batch of dirty pages.
                        region.write_all_to(
                            MemoryRegionAddress(dirty_batch_start),
                            writer,
                            write_size,
                        )?;
                        write_size = 0;
                    }
                }
            }

            if write_size > 0 {
                region.write_all_to(MemoryRegionAddress(dirty_batch_start), writer, write_size)?;
            }

            writer_offset += region.len();
            Ok(())
        })
        .map_err(Error::WriteMemory)
    }

    /// Creates a GuestMemoryMmap given a `file` containing the data
    /// and a `state` containing mapping information.
    fn restore(
        file: &File,
        state: &GuestMemoryState,
        track_dirty_pages: bool,
    ) -> std::result::Result<Self, Error> {
        let mut mmap_regions = Vec::new();
        for region in state.regions.iter() {
            let mmap_region = MmapRegion::build(
                Some(FileOffset::new(
                    file.try_clone().map_err(Error::FileHandle)?,
                    region.offset,
                )),
                region.size,
                libc::PROT_READ | libc::PROT_WRITE,
                libc::MAP_NORESERVE | libc::MAP_PRIVATE,
            )
            .map(|r| {
                let mut region = GuestRegionMmap::new(r, GuestAddress(region.base_address))?;
                if track_dirty_pages {
                    region.enable_dirty_page_tracking();
                }
                Ok(region)
            })
            .map_err(Error::CreateRegion)?
            .map_err(Error::CreateMemory)?;

            mmap_regions.push(mmap_region);
        }

        Ok(Self::from_regions(mmap_regions).map_err(Error::CreateMemory)?)
    }
}

#[cfg(test)]
mod tests {
    use std::collections::HashMap;

    use super::*;
    use utils::tempfile::TempFile;
    use vm_memory::GuestAddress;

    #[test]
    fn test_describe_state() {
        let page_size: usize = sysconf::page::pagesize();

        // Two regions of one page each, with a one page gap between them.
        let mem_regions = [
            (GuestAddress(0), page_size),
            (GuestAddress(page_size as u64 * 2), page_size),
        ];
        let guest_memory = GuestMemoryMmap::from_ranges(&mem_regions[..]).unwrap();

        let expected_memory_state = GuestMemoryState {
            regions: vec![
                GuestMemoryRegionState {
                    base_address: 0,
                    size: page_size,
                    offset: 0,
                },
                GuestMemoryRegionState {
                    base_address: page_size as u64 * 2,
                    size: page_size,
                    offset: page_size as u64,
                },
            ],
        };

        let actual_memory_state = guest_memory.describe();
        assert_eq!(expected_memory_state, actual_memory_state);

        // Two regions of three pages each, with a one page gap between them.
        let mem_regions = [
            (GuestAddress(0), page_size * 3),
            (GuestAddress(page_size as u64 * 4), page_size * 3),
        ];
        let guest_memory = GuestMemoryMmap::from_ranges(&mem_regions[..]).unwrap();

        let expected_memory_state = GuestMemoryState {
            regions: vec![
                GuestMemoryRegionState {
                    base_address: 0,
                    size: page_size * 3,
                    offset: 0,
                },
                GuestMemoryRegionState {
                    base_address: page_size as u64 * 4,
                    size: page_size * 3,
                    offset: page_size as u64 * 3,
                },
            ],
        };

        let actual_memory_state = guest_memory.describe();
        assert_eq!(expected_memory_state, actual_memory_state);
    }

    #[test]
    fn test_restore_memory() {
        let page_size: usize = sysconf::page::pagesize();

        // Two regions of two pages each, with a one page gap between them.
        let mem_regions = [
            (GuestAddress(0), page_size * 2),
            (GuestAddress(page_size as u64 * 3), page_size * 2),
        ];
        let guest_memory = GuestMemoryMmap::from_ranges(&mem_regions[..]).unwrap();

        // Fill the first region with 1s and the second with 2s.
        let first_region = vec![1u8; page_size * 2];
        guest_memory
            .write(&first_region[..], GuestAddress(0))
            .unwrap();

        let second_region = vec![2u8; page_size * 2];
        guest_memory
            .write(&second_region[..], GuestAddress(page_size as u64 * 3))
            .unwrap();

        let memory_state = guest_memory.describe();

        // Case 1: dump the full memory.
        {
            let memory_file = TempFile::new().unwrap();
            guest_memory.dump(&mut memory_file.as_file()).unwrap();

            let restored_guest_memory =
                GuestMemoryMmap::restore(&memory_file.as_file(), &memory_state, false).unwrap();

            // Check that the region contents are the same.
            let mut actual_region = vec![0u8; page_size * 2];
            restored_guest_memory
                .read(&mut actual_region.as_mut_slice(), GuestAddress(0))
                .unwrap();
            assert_eq!(first_region, actual_region);

            restored_guest_memory
                .read(
                    &mut actual_region.as_mut_slice(),
                    GuestAddress(page_size as u64 * 3),
                )
                .unwrap();
            assert_eq!(second_region, actual_region);
        }

        // Case 2: dump only the dirty pages.
        {
            // First region pages: [dirty, clean]
            // Second region pages: [clean, dirty]
            let mut dirty_bitmap: DirtyBitmap = HashMap::new();
            dirty_bitmap.insert(0, vec![0b01; 1]);
            dirty_bitmap.insert(1, vec![0b10; 1]);

            let file = TempFile::new().unwrap();
            guest_memory
                .dump_dirty(&mut file.as_file(), &dirty_bitmap)
                .unwrap();

            let restored_guest_memory =
                GuestMemoryMmap::restore(&file.as_file(), &memory_state, true).unwrap();

            // Check that only the dirty pages have been restored.
            let zeros = vec![0u8; page_size];
            let ones = vec![1u8; page_size];
            let twos = vec![2u8; page_size];
            let expected_first_region = [ones.as_slice(), zeros.as_slice()].concat();
            let expected_second_region = [zeros.as_slice(), twos.as_slice()].concat();

            let mut actual_region = vec![0u8; page_size * 2];
            restored_guest_memory
                .read(&mut actual_region.as_mut_slice(), GuestAddress(0))
                .unwrap();
            assert_eq!(expected_first_region, actual_region);

            restored_guest_memory
                .read(
                    &mut actual_region.as_mut_slice(),
                    GuestAddress(page_size as u64 * 3),
                )
                .unwrap();
            assert_eq!(expected_second_region, actual_region);
        }
    }
}
<commit_msg>Updated dump_dirty to include firecracker bitmap<commit_after>// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

//! Defines functionality for creating guest memory snapshots.

// Currently only used on x86_64.
#![cfg(target_arch = "x86_64")]

use std::fmt::{Display, Formatter};
use std::fs::File;
use std::io::SeekFrom;

use versionize::{VersionMap, Versionize, VersionizeResult};
use versionize_derive::Versionize;
use vm_memory::{
    Bytes, FileOffset, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryMmap,
    GuestMemoryRegion, GuestRegionMmap, MemoryRegionAddress, MmapRegion,
};

use crate::DirtyBitmap;

/// State of a guest memory region saved to file/buffer.
#[derive(Debug, PartialEq, Versionize)]
pub struct GuestMemoryRegionState {
    /// Base address.
    pub base_address: u64,
    /// Region size.
    pub size: usize,
    /// Offset in file/buffer where the region is saved.
    pub offset: u64,
}

/// Guest memory state.
#[derive(Debug, Default, PartialEq, Versionize)]
pub struct GuestMemoryState {
    /// List of regions.
    pub regions: Vec<GuestMemoryRegionState>,
}

/// Defines the interface for snapshotting memory.
pub trait SnapshotMemory
where
    Self: Sized,
{
    /// Describes GuestMemoryMmap through a GuestMemoryState struct.
    fn describe(&self) -> GuestMemoryState;
    /// Dumps all contents of GuestMemoryMmap to a writer.
    fn dump<T: std::io::Write>(&self, writer: &mut T) -> std::result::Result<(), Error>;
    /// Dumps all pages of GuestMemoryMmap present in `dirty_bitmap` to a writer.
    fn dump_dirty<T: std::io::Write + std::io::Seek>(
        &self,
        writer: &mut T,
        dirty_bitmap: &DirtyBitmap,
    ) -> std::result::Result<(), Error>;
    /// Creates a GuestMemoryMmap given a `file` containing the data
    /// and a `state` containing mapping information.
    fn restore(
        file: &File,
        state: &GuestMemoryState,
        track_dirty_pages: bool,
    ) -> std::result::Result<Self, Error>;
}

/// Errors associated with dumping guest memory to file.
#[derive(Debug)]
pub enum Error {
    /// Cannot access file.
    FileHandle(std::io::Error),
    /// Cannot create memory.
    CreateMemory(vm_memory::Error),
    /// Cannot create region.
    CreateRegion(vm_memory::mmap::MmapRegionError),
    /// Cannot dump memory.
    WriteMemory(GuestMemoryError),
}

impl Display for Error {
    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
        use self::Error::*;
        match self {
            FileHandle(err) => write!(f, "Cannot access file: {:?}", err),
            CreateMemory(err) => write!(f, "Cannot create memory: {:?}", err),
            CreateRegion(err) => write!(f, "Cannot create memory region: {:?}", err),
            WriteMemory(err) => write!(f, "Cannot dump memory: {:?}", err),
        }
    }
}

impl SnapshotMemory for GuestMemoryMmap {
    /// Describes GuestMemoryMmap through a GuestMemoryState struct.
    fn describe(&self) -> GuestMemoryState {
        let mut guest_memory_state = GuestMemoryState::default();
        let mut offset = 0;
        let _: std::result::Result<(), ()> = self.with_regions_mut(|_, region| {
            guest_memory_state.regions.push(GuestMemoryRegionState {
                base_address: region.start_addr().0,
                size: region.len() as usize,
                offset,
            });

            offset += region.len();
            Ok(())
        });
        guest_memory_state
    }

    /// Dumps all contents of GuestMemoryMmap to a writer.
    fn dump<T: std::io::Write>(&self, writer: &mut T) -> std::result::Result<(), Error> {
        self.with_regions_mut(|_, region| {
            region.write_all_to(MemoryRegionAddress(0), writer, region.len() as usize)
        })
        .map_err(Error::WriteMemory)
    }

    /// Dumps all pages of GuestMemoryMmap present in `dirty_bitmap` to a writer.
    fn dump_dirty<T: std::io::Write + std::io::Seek>(
        &self,
        writer: &mut T,
        dirty_bitmap: &DirtyBitmap,
    ) -> std::result::Result<(), Error> {
        let page_size = sysconf::page::pagesize();
        let mut writer_offset = 0;

        self.with_regions_mut(|slot, region| {
            let kvm_bitmap = dirty_bitmap.get(&slot).unwrap();
            let firecracker_bitmap = region.dirty_bitmap().unwrap();
            let mut write_size = 0;
            let mut dirty_batch_start: u64 = 0;

            for (i, v) in kvm_bitmap.iter().enumerate() {
                for j in 0..64 {
                    let is_kvm_page_dirty = ((v >> j) & 1u64) != 0u64;
                    let page_offset = ((i * 64) + j) * page_size;
                    let is_firecracker_page_dirty = firecracker_bitmap.is_addr_set(page_offset);
                    if is_kvm_page_dirty || is_firecracker_page_dirty {
                        // We are at the start of a new batch of dirty pages.
                        if write_size == 0 {
                            // Seek forward over the unmodified pages.
                            writer
                                .seek(SeekFrom::Start(writer_offset + page_offset as u64))
                                .unwrap();
                            dirty_batch_start = page_offset as u64;
                        }
                        write_size += page_size;
                    } else if write_size > 0 {
                        // We are at the end of a batch of dirty pages.
                        region.write_all_to(
                            MemoryRegionAddress(dirty_batch_start),
                            writer,
                            write_size,
                        )?;
                        write_size = 0;
                    }
                }
            }

            if write_size > 0 {
                region.write_all_to(MemoryRegionAddress(dirty_batch_start), writer, write_size)?;
            }

            writer_offset += region.len();
            firecracker_bitmap.reset();

            Ok(())
        })
        .map_err(Error::WriteMemory)
    }

    /// Creates a GuestMemoryMmap given a `file` containing the data
    /// and a `state` containing mapping information.
    fn restore(
        file: &File,
        state: &GuestMemoryState,
        track_dirty_pages: bool,
    ) -> std::result::Result<Self, Error> {
        let mut mmap_regions = Vec::new();
        for region in state.regions.iter() {
            let mmap_region = MmapRegion::build(
                Some(FileOffset::new(
                    file.try_clone().map_err(Error::FileHandle)?,
                    region.offset,
                )),
                region.size,
                libc::PROT_READ | libc::PROT_WRITE,
                libc::MAP_NORESERVE | libc::MAP_PRIVATE,
            )
            .map(|r| {
                let mut region = GuestRegionMmap::new(r, GuestAddress(region.base_address))?;
                if track_dirty_pages {
                    region.enable_dirty_page_tracking();
                }
                Ok(region)
            })
            .map_err(Error::CreateRegion)?
            .map_err(Error::CreateMemory)?;

            mmap_regions.push(mmap_region);
        }

        Ok(Self::from_regions(mmap_regions).map_err(Error::CreateMemory)?)
    }
}

#[cfg(test)]
mod tests {
    use std::collections::HashMap;

    use super::*;
    use std::io::{Read, Seek};
    use utils::tempfile::TempFile;
    use vm_memory::GuestAddress;

    #[test]
    fn test_describe_state() {
        let page_size: usize = sysconf::page::pagesize();

        // Two regions of one page each, with a one page gap between them.
        let mem_regions = [
            (GuestAddress(0), page_size),
            (GuestAddress(page_size as u64 * 2), page_size),
        ];
        let guest_memory = GuestMemoryMmap::from_ranges(&mem_regions[..]).unwrap();

        let expected_memory_state = GuestMemoryState {
            regions: vec![
                GuestMemoryRegionState {
                    base_address: 0,
                    size: page_size,
                    offset: 0,
                },
                GuestMemoryRegionState {
                    base_address: page_size as u64 * 2,
                    size: page_size,
                    offset: page_size as u64,
                },
            ],
        };

        let actual_memory_state = guest_memory.describe();
        assert_eq!(expected_memory_state, actual_memory_state);

        // Two regions of three pages each, with a one page gap between them.
        let mem_regions = [
            (GuestAddress(0), page_size * 3),
            (GuestAddress(page_size as u64 * 4), page_size * 3),
        ];
        let guest_memory = GuestMemoryMmap::from_ranges(&mem_regions[..]).unwrap();

        let expected_memory_state = GuestMemoryState {
            regions: vec![
                GuestMemoryRegionState {
                    base_address: 0,
                    size: page_size * 3,
                    offset: 0,
                },
                GuestMemoryRegionState {
                    base_address: page_size as u64 * 4,
                    size: page_size * 3,
                    offset: page_size as u64 * 3,
                },
            ],
        };

        let actual_memory_state = guest_memory.describe();
        assert_eq!(expected_memory_state, actual_memory_state);
    }

    #[test]
    fn test_restore_memory() {
        let page_size: usize = sysconf::page::pagesize();

        // Two regions of two pages each, with a one page gap between them.
        let mem_regions = [
            (GuestAddress(0), page_size * 2),
            (GuestAddress(page_size as u64 * 3), page_size * 2),
        ];
        let guest_memory = GuestMemoryMmap::from_ranges_with_tracking(&mem_regions[..]).unwrap();
        // Check that Firecracker bitmap is clean.
        let _res: std::result::Result<(), Error> = guest_memory.with_regions(|_, r| {
            assert!(!r.dirty_bitmap().unwrap().is_bit_set(0));
            assert!(!r.dirty_bitmap().unwrap().is_bit_set(1));
            Ok(())
        });

        // Fill the first region with 1s and the second with 2s.
        let first_region = vec![1u8; page_size * 2];
        guest_memory
            .write(&first_region[..], GuestAddress(0))
            .unwrap();

        let second_region = vec![2u8; page_size * 2];
        guest_memory
            .write(&second_region[..], GuestAddress(page_size as u64 * 3))
            .unwrap();

        let memory_state = guest_memory.describe();

        // Case 1: dump the full memory.
        {
            let memory_file = TempFile::new().unwrap();
            guest_memory.dump(&mut memory_file.as_file()).unwrap();

            let restored_guest_memory =
                GuestMemoryMmap::restore(&memory_file.as_file(), &memory_state, false).unwrap();

            // Check that the region contents are the same.
            let mut actual_region = vec![0u8; page_size * 2];
            restored_guest_memory
                .read(&mut actual_region.as_mut_slice(), GuestAddress(0))
                .unwrap();
            assert_eq!(first_region, actual_region);

            restored_guest_memory
                .read(
                    &mut actual_region.as_mut_slice(),
                    GuestAddress(page_size as u64 * 3),
                )
                .unwrap();
            assert_eq!(second_region, actual_region);
        }

        // Case 2: dump only the dirty pages.
        {
            // KVM Bitmap
            // First region pages: [dirty, clean]
            // Second region pages: [clean, dirty]
            let mut dirty_bitmap: DirtyBitmap = HashMap::new();
            dirty_bitmap.insert(0, vec![0b01; 1]);
            dirty_bitmap.insert(1, vec![0b10; 1]);

            let file = TempFile::new().unwrap();
            guest_memory
                .dump_dirty(&mut file.as_file(), &dirty_bitmap)
                .unwrap();

            // We can restore from this because this is the first dirty dump.
            let restored_guest_memory =
                GuestMemoryMmap::restore(&file.as_file(), &memory_state, false).unwrap();

            // Check that the region contents are the same.
            let mut actual_region = vec![0u8; page_size * 2];
            restored_guest_memory
                .read(&mut actual_region.as_mut_slice(), GuestAddress(0))
                .unwrap();
            assert_eq!(first_region, actual_region);

            restored_guest_memory
                .read(
                    &mut actual_region.as_mut_slice(),
                    GuestAddress(page_size as u64 * 3),
                )
                .unwrap();
            assert_eq!(second_region, actual_region);

            // Dirty the memory and dump again
            let file = TempFile::new().unwrap();
            let mut reader = file.as_file();
            let zeros = vec![0u8; page_size];
            let ones = vec![1u8; page_size];
            let twos = vec![2u8; page_size];

            // Firecracker Bitmap
            // First region pages: [dirty, clean]
            // Second region pages: [clean, clean]
            guest_memory
                .write(&twos[..], GuestAddress(page_size as u64))
                .unwrap();

            guest_memory.dump_dirty(&mut reader, &dirty_bitmap).unwrap();

            // Check that only the dirty regions are dumped.
            let mut diff_file_content = Vec::new();
            let expected_first_region = [
                ones.as_slice(),
                twos.as_slice(),
                zeros.as_slice(),
                twos.as_slice(),
            ]
            .concat();
            reader.seek(SeekFrom::Start(0)).unwrap();
            reader.read_to_end(&mut diff_file_content).unwrap();
            assert_eq!(expected_first_region, diff_file_content);
        }
    }
}
<|endoftext|>


<commit_before>//! Implements building a CMake-based C++ library.

use crate::cpp_build_config::CppBuildConfigData;
use crate::cpp_build_config::CppBuildPaths;
use crate::cpp_build_config::CppLibraryType;
use crate::errors::{err_msg, Result};
use crate::file_utils::{create_dir_all, path_to_str};
use crate::target;
use crate::utils::run_command;
use crate::utils::run_command_and_capture_output;
use crate::utils::CommandOutput;
use crate::utils::MapIfOk;
use itertools::Itertools;
use log::debug;
use std::fmt::Display;
use std::path::{Path, PathBuf};
use std::process::Command;

/// A CMake variable with a name and a value.
#[derive(Debug, Clone)]
pub struct CMakeVar {
    pub name: String,
    pub value: String,
}
impl CMakeVar {
    /// Creates a new variable.
    pub fn new<S1: Into<String>, S2: Into<String>>(name: S1, value: S2) -> CMakeVar {
        CMakeVar {
            name: name.into(),
            value: value.into(),
        }
    }
    /// Creates a new variable containing a list of values.
    pub fn new_list<I, S, L>(name: S, values: L) -> Result<CMakeVar>
    where
        S: Display + Into<String>,
        I: AsRef<str> + Display,
        L: IntoIterator<Item = I>,
    {
        let value = values
            .into_iter()
            .map_if_ok(|s| -> Result<_> {
                if s.as_ref().contains(';') {
                    Err(err_msg(format!(
                        "can't pass value to cmake because ';' symbol is reserved: {}",
                        s.as_ref()
                    )))
                } else {
                    Ok(s)
                }
            })?
            .into_iter()
            .join(";");
        Ok(CMakeVar::new(name, value))
    }

    /// Creates a new variable containing a list of paths.
    pub fn new_path_list<I, S, L>(name: S, paths: L) -> Result<CMakeVar>
    where
        S: Into<String> + Display,
        I: AsRef<Path>,
        L: IntoIterator<Item = I>,
    {
        CMakeVar::new_list(
            name,
            paths
                .into_iter()
                .map_if_ok(|x| path_to_str(x.as_ref()).map(|x| x.to_string()))?,
        )
    }
}

/// CMake build type (Debug or Release)
#[derive(Debug, Clone)]
pub enum BuildType {
    Debug,
    Release,
}

/// Implements building a CMake-based C++ library.
/// Construct a value and call `run()` to execute building.
#[derive(Debug, Clone)]
pub struct CppLibBuilder {
    /// Path to the source directory containing CMake config file
    pub cmake_source_dir: PathBuf,
    /// Path to the build directory (may not exist before building)
    pub build_dir: PathBuf,
    /// Path to the install directory (may not exist before building)
    pub install_dir: Option<PathBuf>,
    /// Number of threads used to build the library. If `None` is supplied,
    /// number of threads will be detected automatically.
    pub num_jobs: Option<usize>,
    /// CMake build type (Debug or Release)
    pub build_type: BuildType,
    /// Additional variables passed to CMake
    pub cmake_vars: Vec<CMakeVar>,
    pub capture_output: bool,
    pub skip_cmake: bool,
    pub skip_cmake_after_first_run: bool,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CppLibBuilderOutput {
    Success,
    Fail(CommandOutput),
}

impl CppLibBuilder {
    /// Builds the library.
    pub fn run(&mut self) -> Result<CppLibBuilderOutput> {
        if !self.build_dir.exists() {
            create_dir_all(&self.build_dir)?;
        }
        if !self.skip_cmake {
            let mut cmake_command = Command::new("cmake");
            cmake_command
                .arg(&self.cmake_source_dir)
                .current_dir(&self.build_dir);
            let actual_build_type = if target::current_env() == target::Env::Msvc {
                // Rust always links to release version of MSVC runtime, so
                // link will fail if C library is built in debug mode
                BuildType::Release
            } else {
                self.build_type.clone()
            };
            if target::current_os() == target::OS::Windows {
                match target::current_env() {
                    target::Env::Msvc => {
                        cmake_command.arg("-G").arg("NMake Makefiles");
                    }
                    target::Env::Gnu => {
                        cmake_command.arg("-G").arg("MinGW Makefiles");
                    }
                    _ => {}
                }
            }
            let mut actual_cmake_vars = self.cmake_vars.clone();
            actual_cmake_vars.push(CMakeVar::new(
                "CMAKE_BUILD_TYPE",
                match actual_build_type {
                    BuildType::Release => "Release",
                    BuildType::Debug => "Debug",
                },
            ));
            if let Some(install_dir) = &self.install_dir {
                actual_cmake_vars.push(CMakeVar::new(
                    "CMAKE_INSTALL_PREFIX",
                    path_to_str(install_dir)?,
                ));
            }

            for var in actual_cmake_vars {
                cmake_command.arg(format!("-D{}={}", var.name, var.value));
            }
            if self.capture_output {
                let output = run_command_and_capture_output(&mut cmake_command)?;
                if !output.is_success() {
                    return Ok(CppLibBuilderOutput::Fail(output));
                }
            } else {
                run_command(&mut cmake_command)?;
            }
        }

        if self.skip_cmake_after_first_run {
            self.skip_cmake = true;
        }

        let mut make_command_name = if target::current_os() == target::OS::Windows {
            match target::current_env() {
                target::Env::Msvc => "nmake",
                target::Env::Gnu => "mingw32-make",
                _ => "make",
            }
        } else {
            "make"
        };

        let mut make_args = Vec::new();
        let num_jobs = if let Some(x) = self.num_jobs {
            x
        } else {
            ::num_cpus::get()
        };
        if target::current_env() == target::Env::Msvc && num_jobs > 1 {
            debug!("Checking for jom");
            if run_command(&mut Command::new("jom").arg("/version")).is_ok() {
                debug!("jom will be used instead of nmake.");
                make_command_name = "jom";
                make_args.push("/J".to_string());
                make_args.push(num_jobs.to_string());
            } else {
                debug!("jom not found in PATH. Using nmake.");
            }
        }
        if target::current_env() != target::Env::Msvc {
            make_args.push(format!("-j{}", num_jobs));
        }
        if self.install_dir.is_some() {
            make_args.push("install".to_string());
        }
        let mut make_command = Command::new(make_command_name);
        make_command.args(&make_args).current_dir(&self.build_dir);
        if self.capture_output {
            let output = run_command_and_capture_output(&mut make_command)?;
            if !output.is_success() {
                return Ok(CppLibBuilderOutput::Fail(output));
            }
        } else {
            run_command(&mut make_command)?;
        }
        Ok(CppLibBuilderOutput::Success)
    }
}

pub fn c2r_cmake_vars(
    cpp_build_config_data: &CppBuildConfigData,
    cpp_build_paths: &CppBuildPaths,
    library_type: Option<&CppLibraryType>,
) -> Result<Vec<CMakeVar>> {
    let mut cmake_vars = Vec::new();
    if let Some(library_type) = library_type {
        cmake_vars.push(CMakeVar::new(
            "C2R_LIBRARY_TYPE",
            match *library_type {
                CppLibraryType::Shared => "SHARED",
                CppLibraryType::Static => "STATIC",
            },
        ));
    }
    cmake_vars.push(CMakeVar::new_path_list(
        "C2R_INCLUDE_PATHS",
        cpp_build_paths.include_paths(),
    )?);
    cmake_vars.push(CMakeVar::new_path_list(
        "C2R_LIB_PATHS",
        cpp_build_paths.lib_paths(),
    )?);
    cmake_vars.push(CMakeVar::new_path_list(
        "C2R_FRAMEWORK_PATHS",
        cpp_build_paths.framework_paths(),
    )?);
    cmake_vars.push(CMakeVar::new_list(
        "C2R_LINKED_LIBS",
        cpp_build_config_data.linked_libs(),
    )?);
    cmake_vars.push(CMakeVar::new_list(
        "C2R_LINKED_FRAMEWORKS",
        cpp_build_config_data.linked_frameworks(),
    )?);
    cmake_vars.push(CMakeVar::new(
        "C2R_COMPILER_FLAGS",
        cpp_build_config_data.compiler_flags().join(" "),
    ));
    Ok(cmake_vars)
}
<commit_msg>run make clean just in case<commit_after>//! Implements building a CMake-based C++ library.

use crate::cpp_build_config::CppBuildConfigData;
use crate::cpp_build_config::CppBuildPaths;
use crate::cpp_build_config::CppLibraryType;
use crate::errors::{err_msg, Result};
use crate::file_utils::{create_dir_all, path_to_str};
use crate::target;
use crate::utils::run_command;
use crate::utils::run_command_and_capture_output;
use crate::utils::CommandOutput;
use crate::utils::MapIfOk;
use itertools::Itertools;
use log::debug;
use std::fmt::Display;
use std::path::{Path, PathBuf};
use std::process::Command;

/// A CMake variable with a name and a value.
#[derive(Debug, Clone)]
pub struct CMakeVar {
    pub name: String,
    pub value: String,
}
impl CMakeVar {
    /// Creates a new variable.
    pub fn new<S1: Into<String>, S2: Into<String>>(name: S1, value: S2) -> CMakeVar {
        CMakeVar {
            name: name.into(),
            value: value.into(),
        }
    }
    /// Creates a new variable containing a list of values.
    pub fn new_list<I, S, L>(name: S, values: L) -> Result<CMakeVar>
    where
        S: Display + Into<String>,
        I: AsRef<str> + Display,
        L: IntoIterator<Item = I>,
    {
        let value = values
            .into_iter()
            .map_if_ok(|s| -> Result<_> {
                if s.as_ref().contains(';') {
                    Err(err_msg(format!(
                        "can't pass value to cmake because ';' symbol is reserved: {}",
                        s.as_ref()
                    )))
                } else {
                    Ok(s)
                }
            })?
            .into_iter()
            .join(";");
        Ok(CMakeVar::new(name, value))
    }

    /// Creates a new variable containing a list of paths.
    pub fn new_path_list<I, S, L>(name: S, paths: L) -> Result<CMakeVar>
    where
        S: Into<String> + Display,
        I: AsRef<Path>,
        L: IntoIterator<Item = I>,
    {
        CMakeVar::new_list(
            name,
            paths
                .into_iter()
                .map_if_ok(|x| path_to_str(x.as_ref()).map(|x| x.to_string()))?,
        )
    }
}

/// CMake build type (Debug or Release)
#[derive(Debug, Clone)]
pub enum BuildType {
    Debug,
    Release,
}

/// Implements building a CMake-based C++ library.
/// Construct a value and call `run()` to execute building.
#[derive(Debug, Clone)]
pub struct CppLibBuilder {
    /// Path to the source directory containing CMake config file
    pub cmake_source_dir: PathBuf,
    /// Path to the build directory (may not exist before building)
    pub build_dir: PathBuf,
    /// Path to the install directory (may not exist before building)
    pub install_dir: Option<PathBuf>,
    /// Number of threads used to build the library. If `None` is supplied,
    /// number of threads will be detected automatically.
    pub num_jobs: Option<usize>,
    /// CMake build type (Debug or Release)
    pub build_type: BuildType,
    /// Additional variables passed to CMake
    pub cmake_vars: Vec<CMakeVar>,
    pub capture_output: bool,
    pub skip_cmake: bool,
    pub skip_cmake_after_first_run: bool,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CppLibBuilderOutput {
    Success,
    Fail(CommandOutput),
}

impl CppLibBuilder {
    /// Builds the library.
    pub fn run(&mut self) -> Result<CppLibBuilderOutput> {
        if !self.build_dir.exists() {
            create_dir_all(&self.build_dir)?;
        }
        if !self.skip_cmake {
            let mut cmake_command = Command::new("cmake");
            cmake_command
                .arg(&self.cmake_source_dir)
                .current_dir(&self.build_dir);
            let actual_build_type = if target::current_env() == target::Env::Msvc {
                // Rust always links to release version of MSVC runtime, so
                // link will fail if C library is built in debug mode
                BuildType::Release
            } else {
                self.build_type.clone()
            };
            if target::current_os() == target::OS::Windows {
                match target::current_env() {
                    target::Env::Msvc => {
                        cmake_command.arg("-G").arg("NMake Makefiles");
                    }
                    target::Env::Gnu => {
                        cmake_command.arg("-G").arg("MinGW Makefiles");
                    }
                    _ => {}
                }
            }
            let mut actual_cmake_vars = self.cmake_vars.clone();
            actual_cmake_vars.push(CMakeVar::new(
                "CMAKE_BUILD_TYPE",
                match actual_build_type {
                    BuildType::Release => "Release",
                    BuildType::Debug => "Debug",
                },
            ));
            if let Some(install_dir) = &self.install_dir {
                actual_cmake_vars.push(CMakeVar::new(
                    "CMAKE_INSTALL_PREFIX",
                    path_to_str(install_dir)?,
                ));
            }

            for var in actual_cmake_vars {
                cmake_command.arg(format!("-D{}={}", var.name, var.value));
            }
            if self.capture_output {
                let output = run_command_and_capture_output(&mut cmake_command)?;
                if !output.is_success() {
                    return Ok(CppLibBuilderOutput::Fail(output));
                }
            } else {
                run_command(&mut cmake_command)?;
            }
        }

        if self.skip_cmake_after_first_run {
            self.skip_cmake = true;
        }

        let mut make_command_name = if target::current_os() == target::OS::Windows {
            match target::current_env() {
                target::Env::Msvc => "nmake",
                target::Env::Gnu => "mingw32-make",
                _ => "make",
            }
        } else {
            "make"
        };

        run_command(
            Command::new(&make_command_name)
                .arg("clean")
                .current_dir(&self.build_dir),
        )?;

        let mut make_args = Vec::new();
        let num_jobs = if let Some(x) = self.num_jobs {
            x
        } else {
            ::num_cpus::get()
        };
        if target::current_env() == target::Env::Msvc && num_jobs > 1 {
            debug!("Checking for jom");
            if run_command(&mut Command::new("jom").arg("/version")).is_ok() {
                debug!("jom will be used instead of nmake.");
                make_command_name = "jom";
                make_args.push("/J".to_string());
                make_args.push(num_jobs.to_string());
            } else {
                debug!("jom not found in PATH. Using nmake.");
            }
        }
        if target::current_env() != target::Env::Msvc {
            make_args.push(format!("-j{}", num_jobs));
        }
        if self.install_dir.is_some() {
            make_args.push("install".to_string());
        }
        let mut make_command = Command::new(make_command_name);
        make_command.args(&make_args).current_dir(&self.build_dir);
        if self.capture_output {
            let output = run_command_and_capture_output(&mut make_command)?;
            if !output.is_success() {
                return Ok(CppLibBuilderOutput::Fail(output));
            }
        } else {
            run_command(&mut make_command)?;
        }
        Ok(CppLibBuilderOutput::Success)
    }
}

pub fn c2r_cmake_vars(
    cpp_build_config_data: &CppBuildConfigData,
    cpp_build_paths: &CppBuildPaths,
    library_type: Option<&CppLibraryType>,
) -> Result<Vec<CMakeVar>> {
    let mut cmake_vars = Vec::new();
    if let Some(library_type) = library_type {
        cmake_vars.push(CMakeVar::new(
            "C2R_LIBRARY_TYPE",
            match *library_type {
                CppLibraryType::Shared => "SHARED",
                CppLibraryType::Static => "STATIC",
            },
        ));
    }
    cmake_vars.push(CMakeVar::new_path_list(
        "C2R_INCLUDE_PATHS",
        cpp_build_paths.include_paths(),
    )?);
    cmake_vars.push(CMakeVar::new_path_list(
        "C2R_LIB_PATHS",
        cpp_build_paths.lib_paths(),
    )?);
    cmake_vars.push(CMakeVar::new_path_list(
        "C2R_FRAMEWORK_PATHS",
        cpp_build_paths.framework_paths(),
    )?);
    cmake_vars.push(CMakeVar::new_list(
        "C2R_LINKED_LIBS",
        cpp_build_config_data.linked_libs(),
    )?);
    cmake_vars.push(CMakeVar::new_list(
        "C2R_LINKED_FRAMEWORKS",
        cpp_build_config_data.linked_frameworks(),
    )?);
    cmake_vars.push(CMakeVar::new(
        "C2R_COMPILER_FLAGS",
        cpp_build_config_data.compiler_flags().join(" "),
    ));
    Ok(cmake_vars)
}
<|endoftext|>


<commit_before>use std::io;

/// Circular buffer to store a stream. Read/write boundaries are not preserved.
pub struct StreamBuffer {
    buf: Box<[u8]>,
    head: usize,
    tail: usize,
}

impl StreamBuffer {
    pub fn new(capacity: usize) -> Self {
        Self {
            buf: vec![0; capacity + 1].into_boxed_slice(),
            head: 0,
            tail: 0,
        }
    }

    pub fn write_to<W: io::Write>(&mut self, destination: &mut W) -> io::Result<usize> {
        if self.head > self.tail {
            let source_slice = &self.buf[self.tail..self.head];
            let w = destination.write(source_slice)?;
            self.tail += w;
            Ok(w)
        } else if self.head < self.tail {
            let source_slice = &self.buf[self.tail..];
            let w = destination.write(source_slice)?;
            self.tail = (self.tail + w) % self.buf.len();
            Ok(w)
        } else {
            Ok(0)
        }
    }

    pub fn read_from(&mut self, source: &[u8]) -> io::Result<()> {
        // TODO
        Ok(())
    }
}
<commit_msg>Implement StreamBuffer.read_from()<commit_after>use std::io;

/// Circular buffer to store a stream. Read/write boundaries are not preserved.
pub struct StreamBuffer {
    buf: Box<[u8]>,
    head: usize,
    tail: usize,
}

impl StreamBuffer {
    pub fn new(capacity: usize) -> Self {
        Self {
            buf: vec![0; capacity + 1].into_boxed_slice(),
            head: 0,
            tail: 0,
        }
    }

    pub fn size(&self) -> usize {
        if self.head < self.tail {
            self.head + self.buf.len() - self.tail
        } else {
            self.head - self.tail
        }
    }

    pub fn remaining(&self) -> usize {
        self.buf.len() - self.size()
    }

    pub fn write_to<W: io::Write>(&mut self, destination: &mut W) -> io::Result<usize> {
        if self.head > self.tail {
            let source_slice = &self.buf[self.tail..self.head];
            let w = destination.write(source_slice)?;
            self.tail += w;
            Ok(w)
        } else if self.head < self.tail {
            let source_slice = &self.buf[self.tail..];
            let w = destination.write(source_slice)?;
            self.tail = (self.tail + w) % self.buf.len();
            Ok(w)
        } else {
            // else head == tail, which means empty buffer, nothing to do
            Ok(0)
        }
    }

    pub fn read_from(&mut self, source: &[u8]) {
        assert!(self.remaining() >= source.len(), "StreamBuffer must have enough space, check remaining() space before calling read_from()");
        let source_len = source.len();
        let buf_len = self.buf.len();
        if source_len <= buf_len - self.head {
            let target_slice = &mut self.buf[self.head..self.head + source_len];
            target_slice.copy_from_slice(source);
        } else {
            {
                // fill until the right-end of the buffer
                let target_slice = &mut self.buf[self.head..];
                let source_slice = &source[..source_len - self.head];
                target_slice.copy_from_slice(source_slice);
            }
            // fill the remaining from the left-end of the buffer
            let target_slice = &mut self.buf[..self.head + source_len - buf_len];
            let source_slice = &source[source_len - self.head..];
            target_slice.copy_from_slice(source_slice);
        }
        self.head = (self.head + source_len) % buf_len;
    }
}
<|endoftext|>


<commit_before>extern crate getopts;
extern crate lt;
use getopts::Options;
use std::env;
use std::fs::File;

use lt::sampler::params::LTBlockSamplerParams;
use lt::encode;

fn main () {
  let args: Vec<String> = env::args().collect();
  let exe_name = "encoder";
  let mut options = Options::new();
  let matches = match options.parse(&args[1..]) {
    Ok(m) => { m },
    Err(f) => { panic!(f.to_string()) }
  };
  let input_fn = match matches.free.len() {
    1 => matches.free[0].clone(),
    _ => panic!("Usage: encoder <input filename>")
  };

  let mut f = match File::open(input_fn) {
    Ok(file) => file,
    Err(e) => panic!("{}", e)
  };
  let num_blocks = 11;
  let params = LTBlockSamplerParams::new(num_blocks);
  let mut encoder = encode::LTEncoder::new(params, &mut f);
  for block in encoder.take(10) {
    println!("block");
  }
}
<commit_msg>encoder commandline parsing and parameterization<commit_after>extern crate getopts;
extern crate lt;
use getopts::Options;
use std::env;
use std::fs::File;

use lt::sampler::params::LTBlockSamplerParams;
use lt::encode;

fn main () {
  let args: Vec<String> = env::args().collect();
  let mut options = Options::new();
  options.reqopt("f", "input_file", "File to transmit", "FILE");
  options.reqopt("k", "num_blocks", "Number of blocks to divide the file into for transmission", "");
  options.optopt("c", "c", "C parameter to block sampler", "");
  options.optopt("d", "delta", "Delta parameter to block sampler", "");
  options.optopt("s", "seed", "Seed to the block sampler PRNG", "");

  // Validate args
  let matches = match options.parse(&args[1..]) {
    Ok(m) => { m },
    Err(f) => { panic!(f.to_string()) }
  };

  // Get input file from args
  let input_fn = matches.opt_str("f").unwrap();
  let mut f = match File::open(input_fn) {
    Ok(file) => file,
    Err(e) => panic!("{}", e)
  };

  // Parameterize LT sampler using options
  let num_blocks: u32 = match matches.opt_str("k").unwrap().parse() {
    Ok(k) => k,
    Err(e) => panic!("Error parsing k: {}", e)
  };
  let mut params = LTBlockSamplerParams::new(num_blocks);

  // Add c to opts
  params = match matches.opt_str("c") {
    Some(c) => params.c(c.parse::<f64>().unwrap()),
    None => params
  };
  // Add delta to opts
  params = match matches.opt_str("d") {
    Some(delta) => params.delta(delta.parse::<f64>().unwrap()),
    None => params
  };
  // Add seed to opts
  params = match matches.opt_str("s") {
    Some(s) => params.seed(s.parse::<u32>().unwrap()),
    None => params
  };
  let encoder = encode::LTEncoder::new(params, &mut f);
  for block in encoder.take(10) {
    println!("block {:?} ", block);
  }
}
<|endoftext|>


<commit_before>// Copyright 2013-2014 The rust-url developers.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! [*Unicode IDNA Compatibility Processing*
//! (Unicode Technical Standard #46)](http://www.unicode.org/reports/tr46/)

use self::Mapping::*;
use punycode;
use std::cmp::Ordering::{Equal, Greater, Less};
use unicode_bidi::{bidi_class, BidiClass};
use unicode_normalization::char::is_combining_mark;
use unicode_normalization::UnicodeNormalization;

include!("uts46_mapping_table.rs");

const PUNYCODE_PREFIX: &'static str = "xn--";

#[derive(Debug)]
struct StringTableSlice {
    // Store these as separate fields so the structure will have an
    // alignment of 1 and thus pack better into the Mapping enum, below.
    byte_start_lo: u8,
    byte_start_hi: u8,
    byte_len: u8,
}

fn decode_slice(slice: &StringTableSlice) -> &'static str {
    let lo = slice.byte_start_lo as usize;
    let hi = slice.byte_start_hi as usize;
    let start = (hi << 8) | lo;
    let len = slice.byte_len as usize;
    &STRING_TABLE[start..(start + len)]
}

#[repr(u8)]
#[derive(Debug)]
enum Mapping {
    Valid,
    Ignored,
    Mapped(StringTableSlice),
    Deviation(StringTableSlice),
    Disallowed,
    DisallowedStd3Valid,
    DisallowedStd3Mapped(StringTableSlice),
}

struct Range {
    from: char,
    to: char,
}

fn find_char(codepoint: char) -> &'static Mapping {
    let r = TABLE.binary_search_by(|ref range| {
        if codepoint > range.to {
            Less
        } else if codepoint < range.from {
            Greater
        } else {
            Equal
        }
    });
    r.ok()
        .map(|i| {
            const SINGLE_MARKER: u16 = 1 << 15;

            let x = INDEX_TABLE[i];
            let single = (x & SINGLE_MARKER) != 0;
            let offset = !SINGLE_MARKER & x;

            if single {
                &MAPPING_TABLE[offset as usize]
            } else {
                &MAPPING_TABLE[(offset + (codepoint as u16 - TABLE[i].from as u16)) as usize]
            }
        })
        .unwrap()
}

fn map_char(codepoint: char, config: Config, output: &mut String, errors: &mut Vec<Error>) {
    match *find_char(codepoint) {
        Mapping::Valid => output.push(codepoint),
        Mapping::Ignored => {}
        Mapping::Mapped(ref slice) => output.push_str(decode_slice(slice)),
        Mapping::Deviation(ref slice) => {
            if config.transitional_processing {
                output.push_str(decode_slice(slice))
            } else {
                output.push(codepoint)
            }
        }
        Mapping::Disallowed => {
            errors.push(Error::DissallowedCharacter);
            output.push(codepoint);
        }
        Mapping::DisallowedStd3Valid => {
            if config.use_std3_ascii_rules {
                errors.push(Error::DissallowedByStd3AsciiRules);
            }
            output.push(codepoint)
        }
        Mapping::DisallowedStd3Mapped(ref slice) => {
            if config.use_std3_ascii_rules {
                errors.push(Error::DissallowedMappedInStd3);
            }
            output.push_str(decode_slice(slice))
        }
    }
}

// http://tools.ietf.org/html/rfc5893#section-2
fn passes_bidi(label: &str, is_bidi_domain: bool) -> bool {
    // Rule 0: Bidi Rules apply to Bidi Domain Names: a name with at least one RTL label.  A label
    // is RTL if it contains at least one character of bidi class R, AL or AN.
    if !is_bidi_domain {
        return true;
    }

    let mut chars = label.chars();
    let first_char_class = match chars.next() {
        Some(c) => bidi_class(c),
        None => return true, // empty string
    };

    match first_char_class {
        // LTR label
        BidiClass::L => {
            // Rule 5
            loop {
                match chars.next() {
                    Some(c) => {
                        if !matches!(
                            bidi_class(c),
                            BidiClass::L
                                | BidiClass::EN
                                | BidiClass::ES
                                | BidiClass::CS
                                | BidiClass::ET
                                | BidiClass::ON
                                | BidiClass::BN
                                | BidiClass::NSM
                        ) {
                            return false;
                        }
                    }
                    None => {
                        break;
                    }
                }
            }

            // Rule 6
            // must end in L or EN followed by 0 or more NSM
            let mut rev_chars = label.chars().rev();
            let mut last_non_nsm = rev_chars.next();
            loop {
                match last_non_nsm {
                    Some(c) if bidi_class(c) == BidiClass::NSM => {
                        last_non_nsm = rev_chars.next();
                        continue;
                    }
                    _ => {
                        break;
                    }
                }
            }
            match last_non_nsm {
                Some(c) if bidi_class(c) == BidiClass::L || bidi_class(c) == BidiClass::EN => {}
                Some(_) => {
                    return false;
                }
                _ => {}
            }
        }

        // RTL label
        BidiClass::R | BidiClass::AL => {
            let mut found_en = false;
            let mut found_an = false;

            // Rule 2
            loop {
                match chars.next() {
                    Some(c) => {
                        let char_class = bidi_class(c);

                        if char_class == BidiClass::EN {
                            found_en = true;
                        }
                        if char_class == BidiClass::AN {
                            found_an = true;
                        }

                        if !matches!(
                            char_class,
                            BidiClass::R
                                | BidiClass::AL
                                | BidiClass::AN
                                | BidiClass::EN
                                | BidiClass::ES
                                | BidiClass::CS
                                | BidiClass::ET
                                | BidiClass::ON
                                | BidiClass::BN
                                | BidiClass::NSM
                        ) {
                            return false;
                        }
                    }
                    None => {
                        break;
                    }
                }
            }
            // Rule 3
            let mut rev_chars = label.chars().rev();
            let mut last = rev_chars.next();
            loop {
                // must end in L or EN followed by 0 or more NSM
                match last {
                    Some(c) if bidi_class(c) == BidiClass::NSM => {
                        last = rev_chars.next();
                        continue;
                    }
                    _ => {
                        break;
                    }
                }
            }
            match last {
                Some(c)
                    if matches!(
                        bidi_class(c),
                        BidiClass::R | BidiClass::AL | BidiClass::EN | BidiClass::AN
                    ) => {}
                _ => {
                    return false;
                }
            }

            // Rule 4
            if found_an && found_en {
                return false;
            }
        }

        // Rule 1: Should start with L or R/AL
        _ => {
            return false;
        }
    }

    return true;
}

/// http://www.unicode.org/reports/tr46/#Validity_Criteria
fn validate_full(label: &str, is_bidi_domain: bool, config: Config, errors: &mut Vec<Error>) {
    // V1: Must be in NFC form.
    if label.nfc().ne(label.chars()) {
        errors.push(Error::ValidityCriteria);
    } else {
        validate(label, is_bidi_domain, config, errors);
    }
}

fn validate(label: &str, is_bidi_domain: bool, config: Config, errors: &mut Vec<Error>) {
    let first_char = label.chars().next();
    if first_char == None {
        // Empty string, pass
    }
    // V2: No U+002D HYPHEN-MINUS in both third and fourth positions.
    //
    // NOTE: Spec says that the label must not contain a HYPHEN-MINUS character in both the
    // third and fourth positions. But nobody follows this criteria. See the spec issue below:
    // https://github.com/whatwg/url/issues/53

    // V3: neither begin nor end with a U+002D HYPHEN-MINUS
    else if config.check_hyphens && (label.starts_with("-") || label.ends_with("-")) {
        errors.push(Error::ValidityCriteria);
    }
    // V4: not contain a U+002E FULL STOP
    //
    // Here, label can't contain '.' since the input is from .split('.')

    // V5: not begin with a GC=Mark
    else if is_combining_mark(first_char.unwrap()) {
        errors.push(Error::ValidityCriteria);
    }
    // V6: Check against Mapping Table
    else if label.chars().any(|c| match *find_char(c) {
        Mapping::Valid => false,
        Mapping::Deviation(_) => config.transitional_processing,
        Mapping::DisallowedStd3Valid => config.use_std3_ascii_rules,
        _ => true,
    }) {
        errors.push(Error::ValidityCriteria);
    }
    // V7: ContextJ rules
    //
    // TODO: Implement rules and add *CheckJoiners* flag.

    // V8: Bidi rules
    //
    // TODO: Add *CheckBidi* flag
    else if !passes_bidi(label, is_bidi_domain) {
        errors.push(Error::ValidityCriteria);
    }
}

/// http://www.unicode.org/reports/tr46/#Processing
fn processing(domain: &str, config: Config, errors: &mut Vec<Error>) -> String {
    let mut mapped = String::with_capacity(domain.len());
    for c in domain.chars() {
        map_char(c, config, &mut mapped, errors)
    }
    let mut normalized = String::with_capacity(mapped.len());
    normalized.extend(mapped.nfc());

    // Find out if it's a Bidi Domain Name
    //
    // First, check for literal bidi chars
    let mut is_bidi_domain = domain
        .chars()
        .any(|c| matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::AN));
    if !is_bidi_domain {
        // Then check for punycode-encoded bidi chars
        for label in normalized.split('.') {
            if label.starts_with(PUNYCODE_PREFIX) {
                match punycode::decode_to_string(&label[PUNYCODE_PREFIX.len()..]) {
                    Some(decoded_label) => {
                        if decoded_label.chars().any(|c| {
                            matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::AN)
                        }) {
                            is_bidi_domain = true;
                        }
                    }
                    None => {
                        is_bidi_domain = true;
                    }
                }
            }
        }
    }

    let mut validated = String::new();
    let mut first = true;
    for label in normalized.split('.') {
        if !first {
            validated.push('.');
        }
        first = false;
        if label.starts_with(PUNYCODE_PREFIX) {
            match punycode::decode_to_string(&label[PUNYCODE_PREFIX.len()..]) {
                Some(decoded_label) => {
                    let config = config.transitional_processing(false);
                    validate_full(&decoded_label, is_bidi_domain, config, errors);
                    validated.push_str(&decoded_label)
                }
                None => errors.push(Error::PunycodeError),
            }
        } else {
            // `normalized` is already `NFC` so we can skip that check
            validate(label, is_bidi_domain, config, errors);
            validated.push_str(label)
        }
    }
    validated
}

#[derive(Clone, Copy, Default)]
pub struct Config {
    use_std3_ascii_rules: bool,
    transitional_processing: bool,
    verify_dns_length: bool,
    check_hyphens: bool,
}

impl Config {
    #[inline]
    pub fn use_std3_ascii_rules(mut self, value: bool) -> Self {
        self.use_std3_ascii_rules = value;
        self
    }

    #[inline]
    pub fn transitional_processing(mut self, value: bool) -> Self {
        self.transitional_processing = value;
        self
    }

    #[inline]
    pub fn verify_dns_length(mut self, value: bool) -> Self {
        self.verify_dns_length = value;
        self
    }

    #[inline]
    pub fn check_hyphens(mut self, value: bool) -> Self {
        self.check_hyphens = value;
        self
    }

    /// http://www.unicode.org/reports/tr46/#ToASCII
    pub fn to_ascii(self, domain: &str) -> Result<String, Errors> {
        let mut errors = Vec::new();
        let mut result = String::new();
        let mut first = true;
        for label in processing(domain, self, &mut errors).split('.') {
            if !first {
                result.push('.');
            }
            first = false;
            if label.is_ascii() {
                result.push_str(label);
            } else {
                match punycode::encode_str(label) {
                    Some(x) => {
                        result.push_str(PUNYCODE_PREFIX);
                        result.push_str(&x);
                    }
                    None => errors.push(Error::PunycodeError),
                }
            }
        }

        if self.verify_dns_length {
            let domain = if result.ends_with(".") {
                &result[..result.len() - 1]
            } else {
                &*result
            };
            if domain.len() < 1 || domain.split('.').any(|label| label.len() < 1) {
                errors.push(Error::TooShortForDns)
            }
            if domain.len() > 253 || domain.split('.').any(|label| label.len() > 63) {
                errors.push(Error::TooLongForDns)
            }
        }
        if errors.is_empty() {
            Ok(result)
        } else {
            Err(Errors(errors))
        }
    }

    /// http://www.unicode.org/reports/tr46/#ToUnicode
    pub fn to_unicode(self, domain: &str) -> (String, Result<(), Errors>) {
        let mut errors = Vec::new();
        let domain = processing(domain, self, &mut errors);
        let errors = if errors.is_empty() {
            Ok(())
        } else {
            Err(Errors(errors))
        };
        (domain, errors)
    }
}

#[derive(PartialEq, Eq, Clone, Copy, Debug)]
enum Error {
    PunycodeError,
    ValidityCriteria,
    DissallowedByStd3AsciiRules,
    DissallowedMappedInStd3,
    DissallowedCharacter,
    TooLongForDns,
    TooShortForDns,
}

/// Errors recorded during UTS #46 processing.
///
/// This is opaque for now, only indicating the presence of at least one error.
/// More details may be exposed in the future.
#[derive(Debug)]
pub struct Errors(Vec<Error>);
<commit_msg>Have an explicit `impl Default for Config` in idna<commit_after>// Copyright 2013-2014 The rust-url developers.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! [*Unicode IDNA Compatibility Processing*
//! (Unicode Technical Standard #46)](http://www.unicode.org/reports/tr46/)

use self::Mapping::*;
use punycode;
use std::cmp::Ordering::{Equal, Greater, Less};
use unicode_bidi::{bidi_class, BidiClass};
use unicode_normalization::char::is_combining_mark;
use unicode_normalization::UnicodeNormalization;

include!("uts46_mapping_table.rs");

const PUNYCODE_PREFIX: &'static str = "xn--";

#[derive(Debug)]
struct StringTableSlice {
    // Store these as separate fields so the structure will have an
    // alignment of 1 and thus pack better into the Mapping enum, below.
    byte_start_lo: u8,
    byte_start_hi: u8,
    byte_len: u8,
}

fn decode_slice(slice: &StringTableSlice) -> &'static str {
    let lo = slice.byte_start_lo as usize;
    let hi = slice.byte_start_hi as usize;
    let start = (hi << 8) | lo;
    let len = slice.byte_len as usize;
    &STRING_TABLE[start..(start + len)]
}

#[repr(u8)]
#[derive(Debug)]
enum Mapping {
    Valid,
    Ignored,
    Mapped(StringTableSlice),
    Deviation(StringTableSlice),
    Disallowed,
    DisallowedStd3Valid,
    DisallowedStd3Mapped(StringTableSlice),
}

struct Range {
    from: char,
    to: char,
}

fn find_char(codepoint: char) -> &'static Mapping {
    let r = TABLE.binary_search_by(|ref range| {
        if codepoint > range.to {
            Less
        } else if codepoint < range.from {
            Greater
        } else {
            Equal
        }
    });
    r.ok()
        .map(|i| {
            const SINGLE_MARKER: u16 = 1 << 15;

            let x = INDEX_TABLE[i];
            let single = (x & SINGLE_MARKER) != 0;
            let offset = !SINGLE_MARKER & x;

            if single {
                &MAPPING_TABLE[offset as usize]
            } else {
                &MAPPING_TABLE[(offset + (codepoint as u16 - TABLE[i].from as u16)) as usize]
            }
        })
        .unwrap()
}

fn map_char(codepoint: char, config: Config, output: &mut String, errors: &mut Vec<Error>) {
    match *find_char(codepoint) {
        Mapping::Valid => output.push(codepoint),
        Mapping::Ignored => {}
        Mapping::Mapped(ref slice) => output.push_str(decode_slice(slice)),
        Mapping::Deviation(ref slice) => {
            if config.transitional_processing {
                output.push_str(decode_slice(slice))
            } else {
                output.push(codepoint)
            }
        }
        Mapping::Disallowed => {
            errors.push(Error::DissallowedCharacter);
            output.push(codepoint);
        }
        Mapping::DisallowedStd3Valid => {
            if config.use_std3_ascii_rules {
                errors.push(Error::DissallowedByStd3AsciiRules);
            }
            output.push(codepoint)
        }
        Mapping::DisallowedStd3Mapped(ref slice) => {
            if config.use_std3_ascii_rules {
                errors.push(Error::DissallowedMappedInStd3);
            }
            output.push_str(decode_slice(slice))
        }
    }
}

// http://tools.ietf.org/html/rfc5893#section-2
fn passes_bidi(label: &str, is_bidi_domain: bool) -> bool {
    // Rule 0: Bidi Rules apply to Bidi Domain Names: a name with at least one RTL label.  A label
    // is RTL if it contains at least one character of bidi class R, AL or AN.
    if !is_bidi_domain {
        return true;
    }

    let mut chars = label.chars();
    let first_char_class = match chars.next() {
        Some(c) => bidi_class(c),
        None => return true, // empty string
    };

    match first_char_class {
        // LTR label
        BidiClass::L => {
            // Rule 5
            loop {
                match chars.next() {
                    Some(c) => {
                        if !matches!(
                            bidi_class(c),
                            BidiClass::L
                                | BidiClass::EN
                                | BidiClass::ES
                                | BidiClass::CS
                                | BidiClass::ET
                                | BidiClass::ON
                                | BidiClass::BN
                                | BidiClass::NSM
                        ) {
                            return false;
                        }
                    }
                    None => {
                        break;
                    }
                }
            }

            // Rule 6
            // must end in L or EN followed by 0 or more NSM
            let mut rev_chars = label.chars().rev();
            let mut last_non_nsm = rev_chars.next();
            loop {
                match last_non_nsm {
                    Some(c) if bidi_class(c) == BidiClass::NSM => {
                        last_non_nsm = rev_chars.next();
                        continue;
                    }
                    _ => {
                        break;
                    }
                }
            }
            match last_non_nsm {
                Some(c) if bidi_class(c) == BidiClass::L || bidi_class(c) == BidiClass::EN => {}
                Some(_) => {
                    return false;
                }
                _ => {}
            }
        }

        // RTL label
        BidiClass::R | BidiClass::AL => {
            let mut found_en = false;
            let mut found_an = false;

            // Rule 2
            loop {
                match chars.next() {
                    Some(c) => {
                        let char_class = bidi_class(c);

                        if char_class == BidiClass::EN {
                            found_en = true;
                        }
                        if char_class == BidiClass::AN {
                            found_an = true;
                        }

                        if !matches!(
                            char_class,
                            BidiClass::R
                                | BidiClass::AL
                                | BidiClass::AN
                                | BidiClass::EN
                                | BidiClass::ES
                                | BidiClass::CS
                                | BidiClass::ET
                                | BidiClass::ON
                                | BidiClass::BN
                                | BidiClass::NSM
                        ) {
                            return false;
                        }
                    }
                    None => {
                        break;
                    }
                }
            }
            // Rule 3
            let mut rev_chars = label.chars().rev();
            let mut last = rev_chars.next();
            loop {
                // must end in L or EN followed by 0 or more NSM
                match last {
                    Some(c) if bidi_class(c) == BidiClass::NSM => {
                        last = rev_chars.next();
                        continue;
                    }
                    _ => {
                        break;
                    }
                }
            }
            match last {
                Some(c)
                    if matches!(
                        bidi_class(c),
                        BidiClass::R | BidiClass::AL | BidiClass::EN | BidiClass::AN
                    ) => {}
                _ => {
                    return false;
                }
            }

            // Rule 4
            if found_an && found_en {
                return false;
            }
        }

        // Rule 1: Should start with L or R/AL
        _ => {
            return false;
        }
    }

    return true;
}

/// http://www.unicode.org/reports/tr46/#Validity_Criteria
fn validate_full(label: &str, is_bidi_domain: bool, config: Config, errors: &mut Vec<Error>) {
    // V1: Must be in NFC form.
    if label.nfc().ne(label.chars()) {
        errors.push(Error::ValidityCriteria);
    } else {
        validate(label, is_bidi_domain, config, errors);
    }
}

fn validate(label: &str, is_bidi_domain: bool, config: Config, errors: &mut Vec<Error>) {
    let first_char = label.chars().next();
    if first_char == None {
        // Empty string, pass
    }
    // V2: No U+002D HYPHEN-MINUS in both third and fourth positions.
    //
    // NOTE: Spec says that the label must not contain a HYPHEN-MINUS character in both the
    // third and fourth positions. But nobody follows this criteria. See the spec issue below:
    // https://github.com/whatwg/url/issues/53

    // V3: neither begin nor end with a U+002D HYPHEN-MINUS
    else if config.check_hyphens && (label.starts_with("-") || label.ends_with("-")) {
        errors.push(Error::ValidityCriteria);
    }
    // V4: not contain a U+002E FULL STOP
    //
    // Here, label can't contain '.' since the input is from .split('.')

    // V5: not begin with a GC=Mark
    else if is_combining_mark(first_char.unwrap()) {
        errors.push(Error::ValidityCriteria);
    }
    // V6: Check against Mapping Table
    else if label.chars().any(|c| match *find_char(c) {
        Mapping::Valid => false,
        Mapping::Deviation(_) => config.transitional_processing,
        Mapping::DisallowedStd3Valid => config.use_std3_ascii_rules,
        _ => true,
    }) {
        errors.push(Error::ValidityCriteria);
    }
    // V7: ContextJ rules
    //
    // TODO: Implement rules and add *CheckJoiners* flag.

    // V8: Bidi rules
    //
    // TODO: Add *CheckBidi* flag
    else if !passes_bidi(label, is_bidi_domain) {
        errors.push(Error::ValidityCriteria);
    }
}

/// http://www.unicode.org/reports/tr46/#Processing
fn processing(domain: &str, config: Config, errors: &mut Vec<Error>) -> String {
    let mut mapped = String::with_capacity(domain.len());
    for c in domain.chars() {
        map_char(c, config, &mut mapped, errors)
    }
    let mut normalized = String::with_capacity(mapped.len());
    normalized.extend(mapped.nfc());

    // Find out if it's a Bidi Domain Name
    //
    // First, check for literal bidi chars
    let mut is_bidi_domain = domain
        .chars()
        .any(|c| matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::AN));
    if !is_bidi_domain {
        // Then check for punycode-encoded bidi chars
        for label in normalized.split('.') {
            if label.starts_with(PUNYCODE_PREFIX) {
                match punycode::decode_to_string(&label[PUNYCODE_PREFIX.len()..]) {
                    Some(decoded_label) => {
                        if decoded_label.chars().any(|c| {
                            matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::AN)
                        }) {
                            is_bidi_domain = true;
                        }
                    }
                    None => {
                        is_bidi_domain = true;
                    }
                }
            }
        }
    }

    let mut validated = String::new();
    let mut first = true;
    for label in normalized.split('.') {
        if !first {
            validated.push('.');
        }
        first = false;
        if label.starts_with(PUNYCODE_PREFIX) {
            match punycode::decode_to_string(&label[PUNYCODE_PREFIX.len()..]) {
                Some(decoded_label) => {
                    let config = config.transitional_processing(false);
                    validate_full(&decoded_label, is_bidi_domain, config, errors);
                    validated.push_str(&decoded_label)
                }
                None => errors.push(Error::PunycodeError),
            }
        } else {
            // `normalized` is already `NFC` so we can skip that check
            validate(label, is_bidi_domain, config, errors);
            validated.push_str(label)
        }
    }
    validated
}

#[derive(Clone, Copy)]
pub struct Config {
    use_std3_ascii_rules: bool,
    transitional_processing: bool,
    verify_dns_length: bool,
    check_hyphens: bool,
}

/// The defaults are that of https://url.spec.whatwg.org/#idna
impl Default for Config {
    fn default() -> Self {
        Config {
            use_std3_ascii_rules: false,
            transitional_processing: false,
            check_hyphens: false,
            // check_bidi: true,
            // check_joiners: true,

            // Only use for to_ascii, not to_unicode
            verify_dns_length: false,
        }
    }
}

impl Config {
    #[inline]
    pub fn use_std3_ascii_rules(mut self, value: bool) -> Self {
        self.use_std3_ascii_rules = value;
        self
    }

    #[inline]
    pub fn transitional_processing(mut self, value: bool) -> Self {
        self.transitional_processing = value;
        self
    }

    #[inline]
    pub fn verify_dns_length(mut self, value: bool) -> Self {
        self.verify_dns_length = value;
        self
    }

    #[inline]
    pub fn check_hyphens(mut self, value: bool) -> Self {
        self.check_hyphens = value;
        self
    }

    /// http://www.unicode.org/reports/tr46/#ToASCII
    pub fn to_ascii(self, domain: &str) -> Result<String, Errors> {
        let mut errors = Vec::new();
        let mut result = String::new();
        let mut first = true;
        for label in processing(domain, self, &mut errors).split('.') {
            if !first {
                result.push('.');
            }
            first = false;
            if label.is_ascii() {
                result.push_str(label);
            } else {
                match punycode::encode_str(label) {
                    Some(x) => {
                        result.push_str(PUNYCODE_PREFIX);
                        result.push_str(&x);
                    }
                    None => errors.push(Error::PunycodeError),
                }
            }
        }

        if self.verify_dns_length {
            let domain = if result.ends_with(".") {
                &result[..result.len() - 1]
            } else {
                &*result
            };
            if domain.len() < 1 || domain.split('.').any(|label| label.len() < 1) {
                errors.push(Error::TooShortForDns)
            }
            if domain.len() > 253 || domain.split('.').any(|label| label.len() > 63) {
                errors.push(Error::TooLongForDns)
            }
        }
        if errors.is_empty() {
            Ok(result)
        } else {
            Err(Errors(errors))
        }
    }

    /// http://www.unicode.org/reports/tr46/#ToUnicode
    pub fn to_unicode(self, domain: &str) -> (String, Result<(), Errors>) {
        let mut errors = Vec::new();
        let domain = processing(domain, self, &mut errors);
        let errors = if errors.is_empty() {
            Ok(())
        } else {
            Err(Errors(errors))
        };
        (domain, errors)
    }
}

#[derive(PartialEq, Eq, Clone, Copy, Debug)]
enum Error {
    PunycodeError,
    ValidityCriteria,
    DissallowedByStd3AsciiRules,
    DissallowedMappedInStd3,
    DissallowedCharacter,
    TooLongForDns,
    TooShortForDns,
}

/// Errors recorded during UTS #46 processing.
///
/// This is opaque for now, only indicating the presence of at least one error.
/// More details may be exposed in the future.
#[derive(Debug)]
pub struct Errors(Vec<Error>);
<|endoftext|>


<commit_before>//	MIT License
//
//  Copyright © 2017 Michael J Simms. All rights reserved.
//
//	Permission is hereby granted, free of charge, to any person obtaining a copy
//	of this software and associated documentation files (the "Software"), to deal
//	in the Software without restriction, including without limitation the rights
//	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
//	copies of the Software, and to permit persons to whom the Software is
//	furnished to do so, subject to the following conditions:
//
//	The above copyright notice and this permission notice shall be included in all
//	copies or substantial portions of the Software.
//
//	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
//	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
//	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
//	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
//	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
//	SOFTWARE.

use std::collections::HashSet;
use std::collections::HashMap;

/// Each feature has a name and value.
pub struct Feature {
    name: String,
    value: u64,
}

impl Feature {
    pub fn new (name: &str, value: u64) -> Feature {
        Feature { name: name.to_string(), value: value }
    }

    pub fn set_name(&mut self, name: String) {
        self.name = name;
    }

    pub fn get_name(&self) -> String {
        self.name
    }

    pub fn set_value(&mut self, value: u64) {
        self.value = value;
    }

    pub fn get_value(&self) -> u64 {
        self.value
    }
}

pub type FeatureList = Vec<Feature>;
pub type Uint64Set = HashSet<u64>;
pub type FeatureNameToValuesMap<'a> = HashMap<&'a str, Uint64Set>;

/// This class represents a sample.
/// Each sample has a name and list of features.
pub struct Sample {
    name: String,
    features: FeatureList,
}

impl Sample {
    pub fn new (name: &str) -> Sample {
        Sample { name: name.to_string(), features: Sample::create_feature_list() }
    }

    fn create_feature_list() -> FeatureList {
        let mut v: FeatureList = vec![];
        v
    }

    pub fn add_features(&mut self, features: &mut FeatureList) {
        self.features.append(features);
    }

    pub fn add_feature(&mut self, feature: Feature) {
        self.features.push(feature);
    }

    pub fn features(&self) -> FeatureList {
        self.features
    }
}

pub type SampleList = Vec<Sample>;

/// Tree node, used internally.
struct Node {
    feature_name: String,
    left: NodeLink,
    right: NodeLink,
}

impl Node {
    pub fn new (feature_name: &str) -> Node {
        Node { feature_name: feature_name.to_string(), left: None, right: None }
    }

    fn get_feature_name(&self) -> String {
        self.feature_name
    }

	fn set_left_subtree(&mut self, subtree: NodeLink) {
		self.left = subtree;
	}

    fn get_left_subtree(&self) -> NodeLink {
        self.left
    }

	fn set_right_subtree(&mut self, subtree: NodeLink) {
		self.right = subtree;
	}

    fn get_right_subtree(&self) -> NodeLink {
        self.right
    }
}

pub type NodeLink = Option<Box<Node>>;
pub type NodeList = Vec<Box<Node>>;

/// Isolation Forest implementation.
pub struct Forest<'a> {
    feature_values: FeatureNameToValuesMap<'a>, // Lists each feature and maps it to all unique values in the training set
    trees: NodeList, // The decision trees that comprise the forest
    num_trees_to_create: u32, // The maximum number of trees to create
    sub_sampling_size: u32, // The maximum depth of a tree
}

impl<'a> Forest<'a> {
    pub fn new (num_trees_to_create: u32, sub_sampling_size: u32) -> Forest<'a> {
        Forest { num_trees_to_create: num_trees_to_create, sub_sampling_size: sub_sampling_size, trees: Forest::initialize_trees(), feature_values: Forest::create_feature_name_to_values_map() }
    }

    fn initialize_trees() -> NodeList {
        let mut v: NodeList = vec![];
        v
    }

    fn create_feature_name_to_values_map() -> FeatureNameToValuesMap<'a> {
        let mut m = HashMap::new();
        m
    }

    pub fn add_sample(&self, sample: Sample) {
		// Add each of this sample's features to the list of known features
		// with the corresponding set of unique values.
		let mut features = sample.features();
        for feature in &features {
            let mut found_feature = false;
        }
    }

    pub fn create_tree(&self, feature_values: FeatureNameToValuesMap<'a>, depth: u32) -> NodeLink {
		// Sanity check.
		if feature_values.len() <= 1 {
			return None;
		}

		// If we've exceeded the maximum desired depth, then stop.
		if (self.sub_sampling_size > 0) && (depth >= self.sub_sampling_size) {
			return None;
		}

		// Randomly select a feature.
		let mut selected_feature_index = 0;
		if feature_values.len() > 1 {
			return None;
		}

		// Get the value list to split on.

		// Randomly select a split value.
		let mut split_value_index = 0;

		// Create a tree node to hold the split value.
        let mut tree = Some(Box::new(Node::new("")));

        tree
    }

    pub fn create(&self) {
    	for _i in 0..self.num_trees_to_create {
            let tree = self.create_tree(self.feature_values, 0);
            if tree.len() > 0 {
            }
        }
    }

    fn score_tree(&self, sample: Sample, tree: NodeLink) -> f64 {
        let mut depth = 0.0;
        let mut current_node = tree;
        let mut done = false;

        while !done {
            let mut found_feature = false;
            let current_node_deref = current_node;
            let node_feature_name = current_node_deref.get_feature_name();
            let features = sample.features();

            for feature in features {
                let feature_name = feature.get_name();

                if feature_name == node_feature_name {
					if feature.get_value() < current_node_deref.split_value() {
						current_node = current_node_deref.get_left_subtree();
                    } else {
						current_node = current_node_deref.get_right_subtree();
                    }
                    found_feature = false;
                }
            }
        }
        depth
    }

    pub fn score(&self, sample: Sample) -> f64 {
        let mut score = 0.0;

        if self.trees.len() > 0 {
            for tree in self.trees {
                score += self.score_tree(sample, Some(tree));
            }
            score /= self.trees.len() as f64;
        }
        score
    }
}
<commit_msg>Fixed empty check.<commit_after>//	MIT License
//
//  Copyright © 2017 Michael J Simms. All rights reserved.
//
//	Permission is hereby granted, free of charge, to any person obtaining a copy
//	of this software and associated documentation files (the "Software"), to deal
//	in the Software without restriction, including without limitation the rights
//	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
//	copies of the Software, and to permit persons to whom the Software is
//	furnished to do so, subject to the following conditions:
//
//	The above copyright notice and this permission notice shall be included in all
//	copies or substantial portions of the Software.
//
//	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
//	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
//	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
//	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
//	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
//	SOFTWARE.

use std::collections::HashSet;
use std::collections::HashMap;

/// Each feature has a name and value.
pub struct Feature {
    name: String,
    value: u64,
}

impl Feature {
    pub fn new (name: &str, value: u64) -> Feature {
        Feature { name: name.to_string(), value: value }
    }

    pub fn set_name(&mut self, name: String) {
        self.name = name;
    }

    pub fn get_name(&self) -> String {
        self.name
    }

    pub fn set_value(&mut self, value: u64) {
        self.value = value;
    }

    pub fn get_value(&self) -> u64 {
        self.value
    }
}

pub type FeatureList = Vec<Feature>;
pub type Uint64Set = HashSet<u64>;
pub type FeatureNameToValuesMap<'a> = HashMap<&'a str, Uint64Set>;

/// This class represents a sample.
/// Each sample has a name and list of features.
pub struct Sample {
    name: String,
    features: FeatureList,
}

impl Sample {
    pub fn new (name: &str) -> Sample {
        Sample { name: name.to_string(), features: Sample::create_feature_list() }
    }

    fn create_feature_list() -> FeatureList {
        let mut v: FeatureList = vec![];
        v
    }

    pub fn add_features(&mut self, features: &mut FeatureList) {
        self.features.append(features);
    }

    pub fn add_feature(&mut self, feature: Feature) {
        self.features.push(feature);
    }

    pub fn features(&self) -> FeatureList {
        self.features
    }
}

pub type SampleList = Vec<Sample>;

/// Tree node, used internally.
struct Node {
    feature_name: String,
    left: NodeLink,
    right: NodeLink,
}

impl Node {
    pub fn new (feature_name: &str) -> Node {
        Node { feature_name: feature_name.to_string(), left: None, right: None }
    }

    fn get_feature_name(&self) -> String {
        self.feature_name
    }

	fn set_left_subtree(&mut self, subtree: NodeLink) {
		self.left = subtree;
	}

    fn get_left_subtree(&self) -> NodeLink {
        self.left
    }

	fn set_right_subtree(&mut self, subtree: NodeLink) {
		self.right = subtree;
	}

    fn get_right_subtree(&self) -> NodeLink {
        self.right
    }
}

pub type NodeLink = Option<Box<Node>>;
pub type NodeList = Vec<Box<Node>>;

/// Isolation Forest implementation.
pub struct Forest<'a> {
    feature_values: FeatureNameToValuesMap<'a>, // Lists each feature and maps it to all unique values in the training set
    trees: NodeList, // The decision trees that comprise the forest
    num_trees_to_create: u32, // The maximum number of trees to create
    sub_sampling_size: u32, // The maximum depth of a tree
}

impl<'a> Forest<'a> {
    pub fn new (num_trees_to_create: u32, sub_sampling_size: u32) -> Forest<'a> {
        Forest { num_trees_to_create: num_trees_to_create, sub_sampling_size: sub_sampling_size, trees: Forest::initialize_trees(), feature_values: Forest::create_feature_name_to_values_map() }
    }

    fn initialize_trees() -> NodeList {
        let mut v: NodeList = vec![];
        v
    }

    fn create_feature_name_to_values_map() -> FeatureNameToValuesMap<'a> {
        let mut m = HashMap::new();
        m
    }

    pub fn add_sample(&self, sample: Sample) {
		// Add each of this sample's features to the list of known features
		// with the corresponding set of unique values.
		let mut features = sample.features();
        for feature in &features {
            let mut found_feature = false;
        }
    }

    pub fn create_tree(&self, feature_values: FeatureNameToValuesMap<'a>, depth: u32) -> NodeLink {
		// Sanity check.
		if feature_values.len() <= 1 {
			return None;
		}

		// If we've exceeded the maximum desired depth, then stop.
		if (self.sub_sampling_size > 0) && (depth >= self.sub_sampling_size) {
			return None;
		}

		// Randomly select a feature.
		let mut selected_feature_index = 0;
		if feature_values.len() > 1 {
			return None;
		}

		// Get the value list to split on.

		// Randomly select a split value.
		let mut split_value_index = 0;

		// Create a tree node to hold the split value.
        let mut tree = Some(Box::new(Node::new("")));

        tree
    }

    pub fn create(&self) {
    	for _i in 0..self.num_trees_to_create {
            let tree = self.create_tree(self.feature_values, 0);
            if !tree.is_none() {
            }
        }
    }

    fn score_tree(&self, sample: Sample, tree: NodeLink) -> f64 {
        let mut depth = 0.0;
        let mut current_node = tree;
        let mut done = false;

        while !done {
            let mut found_feature = false;
            let current_node_deref = current_node;
            let node_feature_name = current_node_deref.get_feature_name();
            let features = sample.features();

            for feature in features {
                let feature_name = feature.get_name();

                if feature_name == node_feature_name {
					if feature.get_value() < current_node_deref.split_value() {
						current_node = current_node_deref.get_left_subtree();
                    } else {
						current_node = current_node_deref.get_right_subtree();
                    }
                    found_feature = false;
                }
            }
        }
        depth
    }

    pub fn score(&self, sample: Sample) -> f64 {
        let mut score = 0.0;

        if self.trees.len() > 0 {
            for tree in self.trees {
                score += self.score_tree(sample, Some(tree));
            }
            score /= self.trees.len() as f64;
        }
        score
    }
}
<|endoftext|>


<commit_before>use std::fmt; // Import the `fmt` module.

// Define a structure named `List` containing a `Vec`.
struct List(Vec<i32>);

impl fmt::Display for List {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        // Dereference `self` and create a reference to `vec`
        // via destructuring.
        let List(ref vec) = *self;

        try!(write!(f, "["));

        // Iterate over `vec` in `v` while enumerating the iteration
        // count in `count`.
        for (count, v) in vec.iter().enumerate() {
            // For every element except the first, add a comma
            // before calling `write!`. Use `try!` to return on errors.
            if count != 0 { try!(write!(f, ", ")); }

            try!(write!(f, "{}", v));
        }

        // Close the opened bracket and return a fmt::Result value
        write!(f, "]")
    }
}

fn main() {
    let mut v: Vec<i32> = Vec::new();
    for n in 1..1+10000 { // [1..10000] ~ [1..10001)
        v.push(n);
    }
    println!("{}", List(v));
    let v = List(vec![1, 2, 3]);
    println!("{}", v);
}

<commit_msg>complete activity for 1.2.2.1<commit_after>use std::fmt; // Import the `fmt` module.

// Define a structure named `List` containing a `Vec`.
struct List(Vec<i32>);

impl fmt::Display for List {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        // Dereference `self` and create a reference to `vec`
        // via destructuring.
        let List(ref vec) = *self;

        try!(write!(f, "["));

        // Iterate over `vec` in `v` while enumerating the iteration
        // count in `count`.
        for (count, v) in vec.iter().enumerate() {
            // For every element except the first, add a comma
            // before calling `write!`. Use `try!` to return on errors.
            if count != 0 { try!(write!(f, ", ")); }

            try!(write!(f, "{}: {}", count,v));
        }

        // Close the opened bracket and return a fmt::Result value
        write!(f, "]")
    }
}

fn main() {
    let mut v: Vec<i32> = Vec::new();
    for n in 1..1+10000 { // [1..10000] ~ [1..10001)
        v.push(n);
    }
    println!("{}", List(v));
    let v = List(vec![1, 2, 3]);
    println!("{}", v);
}

<|endoftext|>