<commit_before>// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

//! Defines functionality for creating guest memory snapshots.

// Currently only used on x86_64.
#![cfg(target_arch = "x86_64")]

use std::fmt::{Display, Formatter};
use std::fs::File;
use std::io::SeekFrom;

use versionize::{VersionMap, Versionize, VersionizeResult};
use versionize_derive::Versionize;
use vm_memory::{
    Bytes, FileOffset, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryMmap,
    GuestMemoryRegion, GuestRegionMmap, MemoryRegionAddress, MmapRegion,
};

use crate::DirtyBitmap;

/// State of a guest memory region saved to file/buffer.
#[derive(Debug, PartialEq, Versionize)]
pub struct GuestMemoryRegionState {
    /// Base address.
    pub base_address: u64,
    /// Region size.
    pub size: usize,
    /// Offset in file/buffer where the region is saved.
    pub offset: u64,
}

/// Guest memory state.
#[derive(Debug, Default, PartialEq, Versionize)]
pub struct GuestMemoryState {
    /// List of regions.
    pub regions: Vec<GuestMemoryRegionState>,
}

/// Defines the interface for snapshotting memory.
pub trait SnapshotMemory
where
    Self: Sized,
{
    /// Describes GuestMemoryMmap through a GuestMemoryState struct.
    fn describe(&self) -> GuestMemoryState;
    /// Dumps all contents of GuestMemoryMmap to a writer.
    fn dump<T: std::io::Write>(&self, writer: &mut T) -> std::result::Result<(), Error>;
    /// Dumps all pages of GuestMemoryMmap present in `dirty_bitmap` to a writer.
    fn dump_dirty<T: std::io::Write + std::io::Seek>(
        &self,
        writer: &mut T,
        dirty_bitmap: &DirtyBitmap,
    ) -> std::result::Result<(), Error>;
    /// Creates a GuestMemoryMmap given a `file` containing the data
    /// and a `state` containing mapping information.
    fn restore(file: &File, state: &GuestMemoryState) -> std::result::Result<Self, Error>;
}

/// Errors associated with dumping guest memory to file.
#[derive(Debug)]
pub enum Error {
    /// Cannot access file.
    FileHandle(std::io::Error),
    /// Cannot create memory.
    CreateMemory(vm_memory::Error),
    /// Cannot create region.
    CreateRegion(vm_memory::mmap::MmapRegionError),
    /// Cannot dump memory.
    WriteMemory(GuestMemoryError),
}

impl Display for Error {
    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
        use self::Error::*;
        match self {
            FileHandle(err) => write!(f, "Cannot access file: {:?}", err),
            CreateMemory(err) => write!(f, "Cannot create memory: {:?}", err),
            CreateRegion(err) => write!(f, "Cannot create memory region: {:?}", err),
            WriteMemory(err) => write!(f, "Cannot dump memory: {:?}", err),
        }
    }
}

impl SnapshotMemory for GuestMemoryMmap {
    /// Describes GuestMemoryMmap through a GuestMemoryState struct.
    fn describe(&self) -> GuestMemoryState {
        let mut guest_memory_state = GuestMemoryState::default();
        let mut offset = 0;
        let _: std::result::Result<(), ()> = self.with_regions_mut(|_, region| {
            guest_memory_state.regions.push(GuestMemoryRegionState {
                base_address: region.start_addr().0,
                size: region.len() as usize,
                offset,
            });

            offset += region.len();
            Ok(())
        });
        guest_memory_state
    }

    /// Dumps all contents of GuestMemoryMmap to a writer.
    fn dump<T: std::io::Write>(&self, writer: &mut T) -> std::result::Result<(), Error> {
        self.with_regions_mut(|_, region| {
            region
                .write_to(MemoryRegionAddress(0), writer, region.len() as usize)
                .map(|_| ())
        })
        .map_err(Error::WriteMemory)
    }

    /// Dumps all pages of GuestMemoryMmap present in `dirty_bitmap` to a writer.
    fn dump_dirty<T: std::io::Write + std::io::Seek>(
        &self,
        writer: &mut T,
        dirty_bitmap: &DirtyBitmap,
    ) -> std::result::Result<(), Error> {
        let page_size = sysconf::page::pagesize();
        let mut writer_offset = 0;

        self.with_regions_mut(|slot, region| {
            let bitmap = dirty_bitmap.get(&slot).unwrap();
            let mut write_size = 0;
            let mut dirty_batch_start: u64 = 0;

            for (i, v) in bitmap.iter().enumerate() {
                for j in 0..64 {
                    let is_dirty_page = ((v >> j) & 1u64) != 0u64;
                    if is_dirty_page {
                        let page_offset = ((i * 64) + j) * page_size;
                        // We are at the start of a new batch of dirty pages.
                        if write_size == 0 {
                            // Seek forward over the unmodified pages.
                            writer
                                .seek(SeekFrom::Start(writer_offset + page_offset as u64))
                                .unwrap();
                            dirty_batch_start = page_offset as u64;
                        }
                        write_size += page_size;
                    } else if write_size > 0 {
                        // We are at the end of a batch of dirty pages.
                        region
                            .write_to(MemoryRegionAddress(dirty_batch_start), writer, write_size)
                            .map(|_| ())?;
                        write_size = 0;
                    }
                }
            }

            if write_size > 0 {
                region
                    .write_to(MemoryRegionAddress(dirty_batch_start), writer, write_size)
                    .map(|_| ())?;
            }

            writer_offset += region.len();
            Ok(())
        })
        .map_err(Error::WriteMemory)
    }

    /// Creates a GuestMemoryMmap given a `file` containing the data
    /// and a `state` containing mapping information.
    fn restore(file: &File, state: &GuestMemoryState) -> std::result::Result<Self, Error> {
        let mut mmap_regions = Vec::new();
        for region in state.regions.iter() {
            let mmap_region = MmapRegion::build(
                Some(FileOffset::new(
                    file.try_clone().map_err(Error::FileHandle)?,
                    region.offset,
                )),
                region.size,
                libc::PROT_READ | libc::PROT_WRITE,
                libc::MAP_NORESERVE | libc::MAP_PRIVATE,
            )
            .map(|r| GuestRegionMmap::new(r, GuestAddress(region.base_address)))
            .map_err(Error::CreateRegion)?
            .map_err(Error::CreateMemory)?;

            mmap_regions.push(mmap_region);
        }

        Ok(Self::from_regions(mmap_regions).map_err(Error::CreateMemory)?)
    }
}

#[cfg(test)]
mod tests {
    use std::collections::HashMap;

    use super::*;
    use utils::tempfile::TempFile;
    use vm_memory::GuestAddress;

    #[test]
    fn test_describe_state() {
        let page_size: usize = sysconf::page::pagesize();

        // Two regions of one page each, with a one page gap between them.
        let mem_regions = [
            (GuestAddress(0), page_size),
            (GuestAddress(page_size as u64 * 2), page_size),
        ];
        let guest_memory = GuestMemoryMmap::from_ranges(&mem_regions[..]).unwrap();

        let expected_memory_state = GuestMemoryState {
            regions: vec![
                GuestMemoryRegionState {
                    base_address: 0,
                    size: page_size,
                    offset: 0,
                },
                GuestMemoryRegionState {
                    base_address: page_size as u64 * 2,
                    size: page_size,
                    offset: page_size as u64,
                },
            ],
        };

        let actual_memory_state = guest_memory.describe();
        assert_eq!(expected_memory_state, actual_memory_state);

        // Two regions of three pages each, with a one page gap between them.
        let mem_regions = [
            (GuestAddress(0), page_size * 3),
            (GuestAddress(page_size as u64 * 4), page_size * 3),
        ];
        let guest_memory = GuestMemoryMmap::from_ranges(&mem_regions[..]).unwrap();

        let expected_memory_state = GuestMemoryState {
            regions: vec![
                GuestMemoryRegionState {
                    base_address: 0,
                    size: page_size * 3,
                    offset: 0,
                },
                GuestMemoryRegionState {
                    base_address: page_size as u64 * 4,
                    size: page_size * 3,
                    offset: page_size as u64 * 3,
                },
            ],
        };

        let actual_memory_state = guest_memory.describe();
        assert_eq!(expected_memory_state, actual_memory_state);
    }

    #[test]
    fn test_restore_memory() {
        let page_size: usize = sysconf::page::pagesize();

        // Two regions of two pages each, with a one page gap between them.
        let mem_regions = [
            (GuestAddress(0), page_size * 2),
            (GuestAddress(page_size as u64 * 3), page_size * 2),
        ];
        let guest_memory = GuestMemoryMmap::from_ranges(&mem_regions[..]).unwrap();

        // Fill the first region with 1s and the second with 2s.
        let first_region = vec![1u8; page_size * 2];
        guest_memory
            .write(&first_region[..], GuestAddress(0))
            .unwrap();

        let second_region = vec![2u8; page_size * 2];
        guest_memory
            .write(&second_region[..], GuestAddress(page_size as u64 * 3))
            .unwrap();

        let memory_state = guest_memory.describe();

        // Case 1: dump the full memory.
        {
            let memory_file = TempFile::new().unwrap();
            guest_memory.dump(&mut memory_file.as_file()).unwrap();

            let restored_guest_memory =
                GuestMemoryMmap::restore(&memory_file.as_file(), &memory_state).unwrap();

            // Check that the region contents are the same.
            let mut actual_region = vec![0u8; page_size * 2];
            restored_guest_memory
                .read(&mut actual_region.as_mut_slice(), GuestAddress(0))
                .unwrap();
            assert_eq!(first_region, actual_region);

            restored_guest_memory
                .read(
                    &mut actual_region.as_mut_slice(),
                    GuestAddress(page_size as u64 * 3),
                )
                .unwrap();
            assert_eq!(second_region, actual_region);
        }

        // Case 2: dump only the dirty pages.
        {
            // First region pages: [dirty, clean]
            // Second region pages: [clean, dirty]
            let mut dirty_bitmap: DirtyBitmap = HashMap::new();
            dirty_bitmap.insert(0, vec![0b01; 1]);
            dirty_bitmap.insert(1, vec![0b10; 1]);

            let file = TempFile::new().unwrap();
            guest_memory
                .dump_dirty(&mut file.as_file(), &dirty_bitmap)
                .unwrap();

            let restored_guest_memory =
                GuestMemoryMmap::restore(&file.as_file(), &memory_state).unwrap();

            // Check that only the dirty pages have been restored.
            let zeros = vec![0u8; page_size];
            let ones = vec![1u8; page_size];
            let twos = vec![2u8; page_size];
            let expected_first_region = [ones.as_slice(), zeros.as_slice()].concat();
            let expected_second_region = [zeros.as_slice(), twos.as_slice()].concat();

            let mut actual_region = vec![0u8; page_size * 2];
            restored_guest_memory
                .read(&mut actual_region.as_mut_slice(), GuestAddress(0))
                .unwrap();
            assert_eq!(expected_first_region, actual_region);

            restored_guest_memory
                .read(
                    &mut actual_region.as_mut_slice(),
                    GuestAddress(page_size as u64 * 3),
                )
                .unwrap();
            assert_eq!(expected_second_region, actual_region);
        }
    }
}
<commit_msg>snapshot: memory dump: write all bytes<commit_after>// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

//! Defines functionality for creating guest memory snapshots.

// Currently only used on x86_64.
#![cfg(target_arch = "x86_64")]

use std::fmt::{Display, Formatter};
use std::fs::File;
use std::io::SeekFrom;

use versionize::{VersionMap, Versionize, VersionizeResult};
use versionize_derive::Versionize;
use vm_memory::{
    Bytes, FileOffset, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryMmap,
    GuestMemoryRegion, GuestRegionMmap, MemoryRegionAddress, MmapRegion,
};

use crate::DirtyBitmap;

/// State of a guest memory region saved to file/buffer.
#[derive(Debug, PartialEq, Versionize)]
pub struct GuestMemoryRegionState {
    /// Base address.
    pub base_address: u64,
    /// Region size.
    pub size: usize,
    /// Offset in file/buffer where the region is saved.
    pub offset: u64,
}

/// Guest memory state.
#[derive(Debug, Default, PartialEq, Versionize)]
pub struct GuestMemoryState {
    /// List of regions.
    pub regions: Vec<GuestMemoryRegionState>,
}

/// Defines the interface for snapshotting memory.
pub trait SnapshotMemory
where
    Self: Sized,
{
    /// Describes GuestMemoryMmap through a GuestMemoryState struct.
    fn describe(&self) -> GuestMemoryState;
    /// Dumps all contents of GuestMemoryMmap to a writer.
    fn dump<T: std::io::Write>(&self, writer: &mut T) -> std::result::Result<(), Error>;
    /// Dumps all pages of GuestMemoryMmap present in `dirty_bitmap` to a writer.
    fn dump_dirty<T: std::io::Write + std::io::Seek>(
        &self,
        writer: &mut T,
        dirty_bitmap: &DirtyBitmap,
    ) -> std::result::Result<(), Error>;
    /// Creates a GuestMemoryMmap given a `file` containing the data
    /// and a `state` containing mapping information.
    fn restore(file: &File, state: &GuestMemoryState) -> std::result::Result<Self, Error>;
}

/// Errors associated with dumping guest memory to file.
#[derive(Debug)]
pub enum Error {
    /// Cannot access file.
    FileHandle(std::io::Error),
    /// Cannot create memory.
    CreateMemory(vm_memory::Error),
    /// Cannot create region.
    CreateRegion(vm_memory::mmap::MmapRegionError),
    /// Cannot dump memory.
    WriteMemory(GuestMemoryError),
}

impl Display for Error {
    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
        use self::Error::*;
        match self {
            FileHandle(err) => write!(f, "Cannot access file: {:?}", err),
            CreateMemory(err) => write!(f, "Cannot create memory: {:?}", err),
            CreateRegion(err) => write!(f, "Cannot create memory region: {:?}", err),
            WriteMemory(err) => write!(f, "Cannot dump memory: {:?}", err),
        }
    }
}

impl SnapshotMemory for GuestMemoryMmap {
    /// Describes GuestMemoryMmap through a GuestMemoryState struct.
    fn describe(&self) -> GuestMemoryState {
        let mut guest_memory_state = GuestMemoryState::default();
        let mut offset = 0;
        let _: std::result::Result<(), ()> = self.with_regions_mut(|_, region| {
            guest_memory_state.regions.push(GuestMemoryRegionState {
                base_address: region.start_addr().0,
                size: region.len() as usize,
                offset,
            });

            offset += region.len();
            Ok(())
        });
        guest_memory_state
    }

    /// Dumps all contents of GuestMemoryMmap to a writer.
    fn dump<T: std::io::Write>(&self, writer: &mut T) -> std::result::Result<(), Error> {
        self.with_regions_mut(|_, region| {
            region.write_all_to(MemoryRegionAddress(0), writer, region.len() as usize)
        })
        .map_err(Error::WriteMemory)
    }

    /// Dumps all pages of GuestMemoryMmap present in `dirty_bitmap` to a writer.
    fn dump_dirty<T: std::io::Write + std::io::Seek>(
        &self,
        writer: &mut T,
        dirty_bitmap: &DirtyBitmap,
    ) -> std::result::Result<(), Error> {
        let page_size = sysconf::page::pagesize();
        let mut writer_offset = 0;

        self.with_regions_mut(|slot, region| {
            let bitmap = dirty_bitmap.get(&slot).unwrap();
            let mut write_size = 0;
            let mut dirty_batch_start: u64 = 0;

            for (i, v) in bitmap.iter().enumerate() {
                for j in 0..64 {
                    let is_dirty_page = ((v >> j) & 1u64) != 0u64;
                    if is_dirty_page {
                        let page_offset = ((i * 64) + j) * page_size;
                        // We are at the start of a new batch of dirty pages.
                        if write_size == 0 {
                            // Seek forward over the unmodified pages.
                            writer
                                .seek(SeekFrom::Start(writer_offset + page_offset as u64))
                                .unwrap();
                            dirty_batch_start = page_offset as u64;
                        }
                        write_size += page_size;
                    } else if write_size > 0 {
                        // We are at the end of a batch of dirty pages.
                        region.write_all_to(
                            MemoryRegionAddress(dirty_batch_start),
                            writer,
                            write_size,
                        )?;
                        write_size = 0;
                    }
                }
            }

            if write_size > 0 {
                region.write_all_to(MemoryRegionAddress(dirty_batch_start), writer, write_size)?;
            }

            writer_offset += region.len();
            Ok(())
        })
        .map_err(Error::WriteMemory)
    }

    /// Creates a GuestMemoryMmap given a `file` containing the data
    /// and a `state` containing mapping information.
    fn restore(file: &File, state: &GuestMemoryState) -> std::result::Result<Self, Error> {
        let mut mmap_regions = Vec::new();
        for region in state.regions.iter() {
            let mmap_region = MmapRegion::build(
                Some(FileOffset::new(
                    file.try_clone().map_err(Error::FileHandle)?,
                    region.offset,
                )),
                region.size,
                libc::PROT_READ | libc::PROT_WRITE,
                libc::MAP_NORESERVE | libc::MAP_PRIVATE,
            )
            .map(|r| GuestRegionMmap::new(r, GuestAddress(region.base_address)))
            .map_err(Error::CreateRegion)?
            .map_err(Error::CreateMemory)?;

            mmap_regions.push(mmap_region);
        }

        Ok(Self::from_regions(mmap_regions).map_err(Error::CreateMemory)?)
    }
}

#[cfg(test)]
mod tests {
    use std::collections::HashMap;

    use super::*;
    use utils::tempfile::TempFile;
    use vm_memory::GuestAddress;

    #[test]
    fn test_describe_state() {
        let page_size: usize = sysconf::page::pagesize();

        // Two regions of one page each, with a one page gap between them.
        let mem_regions = [
            (GuestAddress(0), page_size),
            (GuestAddress(page_size as u64 * 2), page_size),
        ];
        let guest_memory = GuestMemoryMmap::from_ranges(&mem_regions[..]).unwrap();

        let expected_memory_state = GuestMemoryState {
            regions: vec![
                GuestMemoryRegionState {
                    base_address: 0,
                    size: page_size,
                    offset: 0,
                },
                GuestMemoryRegionState {
                    base_address: page_size as u64 * 2,
                    size: page_size,
                    offset: page_size as u64,
                },
            ],
        };

        let actual_memory_state = guest_memory.describe();
        assert_eq!(expected_memory_state, actual_memory_state);

        // Two regions of three pages each, with a one page gap between them.
        let mem_regions = [
            (GuestAddress(0), page_size * 3),
            (GuestAddress(page_size as u64 * 4), page_size * 3),
        ];
        let guest_memory = GuestMemoryMmap::from_ranges(&mem_regions[..]).unwrap();

        let expected_memory_state = GuestMemoryState {
            regions: vec![
                GuestMemoryRegionState {
                    base_address: 0,
                    size: page_size * 3,
                    offset: 0,
                },
                GuestMemoryRegionState {
                    base_address: page_size as u64 * 4,
                    size: page_size * 3,
                    offset: page_size as u64 * 3,
                },
            ],
        };

        let actual_memory_state = guest_memory.describe();
        assert_eq!(expected_memory_state, actual_memory_state);
    }

    #[test]
    fn test_restore_memory() {
        let page_size: usize = sysconf::page::pagesize();

        // Two regions of two pages each, with a one page gap between them.
        let mem_regions = [
            (GuestAddress(0), page_size * 2),
            (GuestAddress(page_size as u64 * 3), page_size * 2),
        ];
        let guest_memory = GuestMemoryMmap::from_ranges(&mem_regions[..]).unwrap();

        // Fill the first region with 1s and the second with 2s.
        let first_region = vec![1u8; page_size * 2];
        guest_memory
            .write(&first_region[..], GuestAddress(0))
            .unwrap();

        let second_region = vec![2u8; page_size * 2];
        guest_memory
            .write(&second_region[..], GuestAddress(page_size as u64 * 3))
            .unwrap();

        let memory_state = guest_memory.describe();

        // Case 1: dump the full memory.
        {
            let memory_file = TempFile::new().unwrap();
            guest_memory.dump(&mut memory_file.as_file()).unwrap();

            let restored_guest_memory =
                GuestMemoryMmap::restore(&memory_file.as_file(), &memory_state).unwrap();

            // Check that the region contents are the same.
            let mut actual_region = vec![0u8; page_size * 2];
            restored_guest_memory
                .read(&mut actual_region.as_mut_slice(), GuestAddress(0))
                .unwrap();
            assert_eq!(first_region, actual_region);

            restored_guest_memory
                .read(
                    &mut actual_region.as_mut_slice(),
                    GuestAddress(page_size as u64 * 3),
                )
                .unwrap();
            assert_eq!(second_region, actual_region);
        }

        // Case 2: dump only the dirty pages.
        {
            // First region pages: [dirty, clean]
            // Second region pages: [clean, dirty]
            let mut dirty_bitmap: DirtyBitmap = HashMap::new();
            dirty_bitmap.insert(0, vec![0b01; 1]);
            dirty_bitmap.insert(1, vec![0b10; 1]);

            let file = TempFile::new().unwrap();
            guest_memory
                .dump_dirty(&mut file.as_file(), &dirty_bitmap)
                .unwrap();

            let restored_guest_memory =
                GuestMemoryMmap::restore(&file.as_file(), &memory_state).unwrap();

            // Check that only the dirty pages have been restored.
            let zeros = vec![0u8; page_size];
            let ones = vec![1u8; page_size];
            let twos = vec![2u8; page_size];
            let expected_first_region = [ones.as_slice(), zeros.as_slice()].concat();
            let expected_second_region = [zeros.as_slice(), twos.as_slice()].concat();

            let mut actual_region = vec![0u8; page_size * 2];
            restored_guest_memory
                .read(&mut actual_region.as_mut_slice(), GuestAddress(0))
                .unwrap();
            assert_eq!(expected_first_region, actual_region);

            restored_guest_memory
                .read(
                    &mut actual_region.as_mut_slice(),
                    GuestAddress(page_size as u64 * 3),
                )
                .unwrap();
            assert_eq!(expected_second_region, actual_region);
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2018 The Exonum Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use rand::{self, Rng};

use super::{NodeHandler, NodeRole, RequestData};
use crypto::PublicKey;
use events::error::LogError;
use events::network::ConnectedPeerAddr;
use helpers::Height;
use messages::{Connect, Message, PeersRequest, Responses, Service, Signed, Status};

impl NodeHandler {
    /// Redirects message to the corresponding `handle_...` function.
    pub fn handle_message(&mut self, msg: Message) {
        match msg {
            Message::Consensus(msg) => self.handle_consensus(msg),
            Message::Requests(ref msg) => self.handle_request(msg),

            Message::Service(Service::Connect(msg)) => self.handle_connect(msg),
            Message::Service(Service::Status(msg)) => self.handle_status(&msg),
            // ignore tx duplication error,
            Message::Service(Service::RawTransaction(msg)) => drop(self.handle_tx(msg)),
            Message::Responses(Responses::BlockResponse(msg)) => {
                self.handle_block(&msg).log_error()
            }
            Message::Responses(Responses::TransactionsResponse(msg)) => {
                self.handle_txs_batch(&msg).log_error()
            }
        }
    }

    /// Handles the `Connected` event. Node's `Connect` message is sent as response
    /// if received `Connect` message is correct.
    pub fn handle_connected(&mut self, address: &ConnectedPeerAddr, connect: Signed<Connect>) {
        info!("Received Connect message from peer: {:?}", address);
        // TODO: use `ConnectInfo` instead of connect-messages. (ECR-1452)
        self.state.add_connection(connect.author(), address.clone());
        self.handle_connect(connect);
    }

    /// Handles the `Disconnected` event. Node will try to connect to that address again if it was
    /// in the validators list.
    pub fn handle_disconnected(&mut self, key: PublicKey) {
        info!("Disconnected from: {}", key);
        self.remove_peer_with_addr(key);
    }

    /// Handles the `UnableConnectToPeer` event. Node will try to connect to that address again
    /// if it was in the validators list.
    pub fn handle_unable_to_connect(&mut self, key: PublicKey) {
        info!("Could not connect to: {}", key);
        self.remove_peer_with_addr(key);
    }

    /// Removes peer from the state and from the cache. Node will try to connect to that address
    /// again if it was in the validators list.
    fn remove_peer_with_addr(&mut self, key: PublicKey) {
        self.state.remove_peer_with_pubkey(&key);
        self.blockchain.remove_peer_with_pubkey(&key);
        let is_validator = self.state.peer_is_validator(&key);
        let in_connect_list = self.state.peer_in_connect_list(&key);
        if is_validator && in_connect_list {
            self.connect(key);
        }
    }

    /// Handles the `Connect` message and connects to a peer as result.
    pub fn handle_connect(&mut self, message: Signed<Connect>) {
        // TODO Add spam protection (ECR-170)
        // TODO: drop connection if checks have failed. (ECR-1837)
        let address = message.pub_addr().to_owned();
        if address == self.state.our_connect_message().pub_addr() {
            trace!("Received Connect with same address as our external_address.");
            return;
        }

        let public_key = message.author();
        if public_key == self.state.our_connect_message().author() {
            trace!("Received Connect with same pub_key as ours.");
            return;
        }

        if !self.state.connect_list().is_peer_allowed(&public_key) {
            error!(
                "Received connect message from {:?} peer which not in ConnectList.",
                public_key
            );
            return;
        }

        // Check if we have another connect message from peer with the given public_key.
        let mut need_connect = true;
        if let Some(saved_message) = self.state.peers().get(&public_key) {
            if saved_message.time() > message.time() {
                error!("Received outdated Connect message from {}", address);
                return;
            } else if saved_message.time() < message.time() {
                need_connect = saved_message.pub_addr() != message.pub_addr();
            } else if saved_message.pub_addr() == message.pub_addr() {
                need_connect = false;
            } else {
                error!("Received weird Connect message from {}", address);
                return;
            }
            if saved_message.pub_addr() != message.pub_addr() {
                info!(
                    "Updating connect list for peer: {} with new addr: {}",
                    public_key,
                    message.pub_addr()
                );
                self.state
                    .connect_list()
                    .update_peer(&public_key, message.pub_addr().to_string())
            }
        }
        self.state.add_peer(public_key, message.clone());
        info!(
            "Received Connect message from {}, {}",
            address, need_connect,
        );
        self.blockchain.save_peer(&public_key, message);
        if need_connect {
            // TODO: reduce double sending of connect message
            info!("Send Connect message to {}", address);
            //TODO: remove responding connect [ECR-2385]
            self.connect(public_key);
        }
    }

    /// Handles the `Status` message. Node sends `BlockRequest` as response if height in the
    /// message is higher than node's height.
    pub fn handle_status(&mut self, msg: &Signed<Status>) {
        let height = self.state.height();
        trace!(
            "HANDLE STATUS: current height = {}, msg height = {}",
            height,
            msg.height()
        );

        if !self.state.connect_list().is_peer_allowed(&msg.author()) {
            error!(
                "Received status message from peer = {:?} which not in ConnectList.",
                msg.author()
            );
            return;
        }

        // Handle message from future height
        if msg.height() > height {
            let peer = msg.author();

            // Check validator height info
            if msg.height() > self.state.node_height(&peer) {
                // Update validator height
                self.state.set_node_height(peer, msg.height());
            }

            // Request block
            self.request(RequestData::Block(height), peer);
        }
    }

    /// Handles the `PeersRequest` message. Node sends `Connect` messages of other peers as result.
    pub fn handle_request_peers(&mut self, msg: &Signed<PeersRequest>) {
        let peers: Vec<Signed<Connect>> =
            self.state.peers().iter().map(|(_, b)| b.clone()).collect();
        trace!(
            "HANDLE REQUEST PEERS: Sending {:?} peers to {:?}",
            peers,
            msg.author()
        );

        for peer in peers {
            self.send_to_peer(msg.author(), peer);
        }
    }

    /// Handles `NodeTimeout::Status`, broadcasts the `Status` message if it isn't outdated as
    /// result.
    pub fn handle_status_timeout(&mut self, height: Height) {
        if self.state.height() == height {
            self.broadcast_status();
            self.add_status_timeout();
        }
    }
    /// Handles `NodeTimeout::PeerExchange`. Node sends the `PeersRequest` to a random peer.
    pub fn handle_peer_exchange_timeout(&mut self) {
        if !self.state.peers().is_empty() {
            let to = self.state.peers().len();
            let gen_peer_id = || -> usize {
                let mut rng = rand::thread_rng();
                rng.gen_range(0, to)
            };

            let peer = self
                .state
                .peers()
                .iter()
                .map(|x| x.1.clone())
                .nth(gen_peer_id())
                .unwrap();
            let peer = peer.clone();
            let msg = PeersRequest::new(&peer.author());
            trace!("Request peers from peer with addr {:?}", peer.pub_addr());
            let message = self.sign_message(msg);
            self.send_to_peer(peer.author(), message);
        }
        self.add_peer_exchange_timeout();
    }
    /// Handles `NodeTimeout::UpdateApiState`.
    /// Node update internal `ApiState` and `NodeRole`.
    pub fn handle_update_api_state_timeout(&mut self) {
        self.api_state.update_node_state(&self.state);
        self.node_role = NodeRole::new(self.state.validator_id());
        self.add_update_api_state_timeout();
    }

    /// Broadcasts the `Status` message to all peers.
    pub fn broadcast_status(&mut self) {
        let hash = self.blockchain.last_hash();
        let status = Status::new(self.state.height(), &hash);
        trace!("Broadcast status: {:?}", status);

        let message = self.sign_message(status);
        self.broadcast(message);
    }
}
<commit_msg>Tiny log improvement (#1087)<commit_after>// Copyright 2018 The Exonum Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use rand::{self, Rng};

use super::{NodeHandler, NodeRole, RequestData};
use crypto::PublicKey;
use events::error::LogError;
use events::network::ConnectedPeerAddr;
use helpers::Height;
use messages::{Connect, Message, PeersRequest, Responses, Service, Signed, Status};

impl NodeHandler {
    /// Redirects message to the corresponding `handle_...` function.
    pub fn handle_message(&mut self, msg: Message) {
        match msg {
            Message::Consensus(msg) => self.handle_consensus(msg),
            Message::Requests(ref msg) => self.handle_request(msg),

            Message::Service(Service::Connect(msg)) => self.handle_connect(msg),
            Message::Service(Service::Status(msg)) => self.handle_status(&msg),
            // ignore tx duplication error,
            Message::Service(Service::RawTransaction(msg)) => drop(self.handle_tx(msg)),
            Message::Responses(Responses::BlockResponse(msg)) => {
                self.handle_block(&msg).log_error()
            }
            Message::Responses(Responses::TransactionsResponse(msg)) => {
                self.handle_txs_batch(&msg).log_error()
            }
        }
    }

    /// Handles the `Connected` event. Node's `Connect` message is sent as response
    /// if received `Connect` message is correct.
    pub fn handle_connected(&mut self, address: &ConnectedPeerAddr, connect: Signed<Connect>) {
        info!("Received Connect message from peer: {:?}", address);
        // TODO: use `ConnectInfo` instead of connect-messages. (ECR-1452)
        self.state.add_connection(connect.author(), address.clone());
        self.handle_connect(connect);
    }

    /// Handles the `Disconnected` event. Node will try to connect to that address again if it was
    /// in the validators list.
    pub fn handle_disconnected(&mut self, key: PublicKey) {
        info!("Disconnected from: {}", key);
        self.remove_peer_with_addr(key);
    }

    /// Handles the `UnableConnectToPeer` event. Node will try to connect to that address again
    /// if it was in the validators list.
    pub fn handle_unable_to_connect(&mut self, key: PublicKey) {
        info!("Could not connect to: {}", key);
        self.remove_peer_with_addr(key);
    }

    /// Removes peer from the state and from the cache. Node will try to connect to that address
    /// again if it was in the validators list.
    fn remove_peer_with_addr(&mut self, key: PublicKey) {
        self.state.remove_peer_with_pubkey(&key);
        self.blockchain.remove_peer_with_pubkey(&key);
        let is_validator = self.state.peer_is_validator(&key);
        let in_connect_list = self.state.peer_in_connect_list(&key);
        if is_validator && in_connect_list {
            self.connect(key);
        }
    }

    /// Handles the `Connect` message and connects to a peer as result.
    pub fn handle_connect(&mut self, message: Signed<Connect>) {
        // TODO Add spam protection (ECR-170)
        // TODO: drop connection if checks have failed. (ECR-1837)
        let address = message.pub_addr().to_owned();
        if address == self.state.our_connect_message().pub_addr() {
            trace!("Received Connect with same address as our external_address.");
            return;
        }

        let public_key = message.author();
        if public_key == self.state.our_connect_message().author() {
            trace!("Received Connect with same pub_key as ours.");
            return;
        }

        if !self.state.connect_list().is_peer_allowed(&public_key) {
            error!(
                "Received connect message from {:?} peer which not in ConnectList.",
                public_key
            );
            return;
        }

        // Check if we have another connect message from peer with the given public_key.
        let mut need_connect = true;
        if let Some(saved_message) = self.state.peers().get(&public_key) {
            if saved_message.time() > message.time() {
                error!("Received outdated Connect message from {}", address);
                return;
            } else if saved_message.time() < message.time() {
                need_connect = saved_message.pub_addr() != message.pub_addr();
            } else if saved_message.pub_addr() == message.pub_addr() {
                need_connect = false;
            } else {
                error!("Received weird Connect message from {}", address);
                return;
            }
            if saved_message.pub_addr() != message.pub_addr() {
                info!(
                    "Updating connect list for peer: {} with new addr: {}",
                    public_key,
                    message.pub_addr()
                );
                self.state
                    .connect_list()
                    .update_peer(&public_key, message.pub_addr().to_string())
            }
        }
        self.state.add_peer(public_key, message.clone());
        info!(
            "Received Connect message from {}. Need to connect: {}",
            address, need_connect,
        );
        self.blockchain.save_peer(&public_key, message);
        if need_connect {
            // TODO: reduce double sending of connect message
            info!("Send Connect message to {}", address);
            //TODO: remove responding connect [ECR-2385]
            self.connect(public_key);
        }
    }

    /// Handles the `Status` message. Node sends `BlockRequest` as response if height in the
    /// message is higher than node's height.
    pub fn handle_status(&mut self, msg: &Signed<Status>) {
        let height = self.state.height();
        trace!(
            "HANDLE STATUS: current height = {}, msg height = {}",
            height,
            msg.height()
        );

        if !self.state.connect_list().is_peer_allowed(&msg.author()) {
            error!(
                "Received status message from peer = {:?} which not in ConnectList.",
                msg.author()
            );
            return;
        }

        // Handle message from future height
        if msg.height() > height {
            let peer = msg.author();

            // Check validator height info
            if msg.height() > self.state.node_height(&peer) {
                // Update validator height
                self.state.set_node_height(peer, msg.height());
            }

            // Request block
            self.request(RequestData::Block(height), peer);
        }
    }

    /// Handles the `PeersRequest` message. Node sends `Connect` messages of other peers as result.
    pub fn handle_request_peers(&mut self, msg: &Signed<PeersRequest>) {
        let peers: Vec<Signed<Connect>> =
            self.state.peers().iter().map(|(_, b)| b.clone()).collect();
        trace!(
            "HANDLE REQUEST PEERS: Sending {:?} peers to {:?}",
            peers,
            msg.author()
        );

        for peer in peers {
            self.send_to_peer(msg.author(), peer);
        }
    }

    /// Handles `NodeTimeout::Status`, broadcasts the `Status` message if it isn't outdated as
    /// result.
    pub fn handle_status_timeout(&mut self, height: Height) {
        if self.state.height() == height {
            self.broadcast_status();
            self.add_status_timeout();
        }
    }
    /// Handles `NodeTimeout::PeerExchange`. Node sends the `PeersRequest` to a random peer.
    pub fn handle_peer_exchange_timeout(&mut self) {
        if !self.state.peers().is_empty() {
            let to = self.state.peers().len();
            let gen_peer_id = || -> usize {
                let mut rng = rand::thread_rng();
                rng.gen_range(0, to)
            };

            let peer = self
                .state
                .peers()
                .iter()
                .map(|x| x.1.clone())
                .nth(gen_peer_id())
                .unwrap();
            let peer = peer.clone();
            let msg = PeersRequest::new(&peer.author());
            trace!("Request peers from peer with addr {:?}", peer.pub_addr());
            let message = self.sign_message(msg);
            self.send_to_peer(peer.author(), message);
        }
        self.add_peer_exchange_timeout();
    }
    /// Handles `NodeTimeout::UpdateApiState`.
    /// Node update internal `ApiState` and `NodeRole`.
    pub fn handle_update_api_state_timeout(&mut self) {
        self.api_state.update_node_state(&self.state);
        self.node_role = NodeRole::new(self.state.validator_id());
        self.add_update_api_state_timeout();
    }

    /// Broadcasts the `Status` message to all peers.
    pub fn broadcast_status(&mut self) {
        let hash = self.blockchain.last_hash();
        let status = Status::new(self.state.height(), &hash);
        trace!("Broadcast status: {:?}", status);

        let message = self.sign_message(status);
        self.broadcast(message);
    }
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

mod cmd_finder;

use std::env;
use std::env::consts;
use std::fs;
use std::path::{Path, PathBuf};

static SHADERC_STATIC_LIB: &str = "shaderc_combined";
static SHADERC_SHARED_LIB: &str = "shaderc_shared";
static SHADERC_STATIC_LIB_FILE: &str = "libshaderc_combined.a";
static SHADERC_STATIC_LIB_FILE_MSVC: &str = "shaderc_combined.lib";

fn sdk_path() -> Option<PathBuf> {
    let target = std::env::var("TARGET").unwrap();
    use std::process::Command;

    // tvOS (and the simulator) could be added here in the future.
    let sdk = if target == "x86_64-apple-ios" || target == "i386-apple-ios" {
        "iphonesimulator"
    } else if target == "aarch64-apple-ios"
        || target == "armv7-apple-ios"
        || target == "armv7s-apple-ios"
    {
        "iphoneos"
    } else {
        return None;
    };

    let output = if let Ok(out) = Command::new("xcrun")
        .args(&["--sdk", sdk, "--show-sdk-path"])
        .output()
    {
        out.stdout
    } else {
        return None;
    };
    let prefix_str = std::str::from_utf8(&output).expect("invalid output from `xcrun`");
    Some(PathBuf::from(prefix_str.trim_end().to_string()))
}

fn build_shaderc(shaderc_dir: &PathBuf, use_ninja: bool, target_os: String) -> PathBuf {
    let mut config = cmake::Config::new(shaderc_dir);
    config
        .profile("Release")
        .define("CMAKE_POSITION_INDEPENDENT_CODE", "ON")
        .define("SPIRV_SKIP_EXECUTABLES", "ON")
        .define("SPIRV_WERROR", "OFF")
        .define("SHADERC_SKIP_TESTS", "ON")
        .define("CMAKE_INSTALL_LIBDIR", "lib");
    if use_ninja {
        config.generator("Ninja");
    }

    if target_os == "ios" {
        if let Some(path) = sdk_path() {
            config.define("CMAKE_OSX_SYSROOT", path);
        }
    }

    config.build()
}

fn build_shaderc_msvc(shaderc_dir: &PathBuf) -> PathBuf {
    let linkage = env::var("CARGO_CFG_TARGET_FEATURE").unwrap_or_default();

    let mut config = cmake::Config::new(shaderc_dir);
    config
        .profile("Release")
        .define("CMAKE_POSITION_INDEPENDENT_CODE", "ON")
        .define("SPIRV_SKIP_EXECUTABLES", "ON")
        .define("SPIRV_WERROR", "OFF")
        .define("SHADERC_SKIP_TESTS", "ON")
        .define("CMAKE_INSTALL_LIBDIR", "lib")
        .generator("Ninja");

    // cmake-rs tries to be clever on Windows by injecting several
    // C/C++ flags, which causes problems. So I have to manually
    // define CMAKE_*_FLAGS_* here to suppress that.
    let config = if linkage.contains("crt-static") {
        // statically-linked CRT
        config
            .define("CMAKE_C_FLAGS", " /nologo /EHsc /MT")
            .define("CMAKE_CXX_FLAGS", " /nologo /EHsc /MT")
            .define("CMAKE_C_FLAGS_RELEASE", " /nologo /EHsc /MT")
            .define("CMAKE_CXX_FLAGS_RELEASE", " /nologo /EHsc /MT")
    } else {
        // dynamically-linked CRT
        config
            .define("CMAKE_C_FLAGS", " /nologo /EHsc /MD")
            .define("CMAKE_CXX_FLAGS", " /nologo /EHsc /MD")
            .define("CMAKE_C_FLAGS_RELEASE", " /nologo /EHsc /MD")
            .define("CMAKE_CXX_FLAGS_RELEASE", " /nologo /EHsc /MD")
            // prevent shaderc's cmake script messes with crt flags
            .define("SHADERC_ENABLE_SHARED_CRT", "ON")
    };

    config.build()
}

fn probe_linux_lib_kind(lib_name: &str, search_dir: &PathBuf) -> Option<&'static str> {
    if search_dir.join(&format!("lib{}.so", lib_name)).exists() {
        Some("dylib")
    } else if search_dir.join(&format!("lib{}.a", lib_name)).exists() {
        Some("static")
    } else {
        None
    }
}

fn main() {
    let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap();
    let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap();
    let config_build_from_source = env::var("CARGO_FEATURE_BUILD_FROM_SOURCE").is_ok();
    let explicit_lib_dir_set = env::var("SHADERC_LIB_DIR").is_ok();

    // Initialize explicit libshaderc search directory first
    let mut search_dir = if let Ok(lib_dir) = env::var("SHADERC_LIB_DIR") {
        println!(
            "cargo:warning=Specified {} to search for shaderc libraries",
            lib_dir
        );
        Some(lib_dir)
    } else {
        None
    };

    // If no explicit path is set and no explicit request is made to build from
    // source, check known locations before falling back to from-source-build
    if search_dir.is_none() && target_os == "linux" && !config_build_from_source {
        println!(
            "cargo:warning=Checking for system installed libraries.  \
             Use --features = build-from-source to disable this behavior"
        );

        // https://wiki.ubuntu.com/MultiarchSpec
        // https://wiki.debian.org/Multiarch/Implementation
        let debian_arch = match env::var("CARGO_CFG_TARGET_ARCH").unwrap() {
            arch if arch == "x86" => "i386".to_owned(),
            arch => arch,
        };
        let debian_triple_path = format!("/usr/lib/{}-linux-gnu/", debian_arch);

        search_dir = if Path::new(&debian_triple_path).exists() {
            // Debian, Ubuntu and their derivatives.
            Some(debian_triple_path)
        } else if env::var("CARGO_CFG_TARGET_ARCH").unwrap() == "x86_64"
            && Path::new("/usr/lib64/").exists()
        {
            // Other distributions running on x86_64 usually use this path.
            Some("/usr/lib64/".to_owned())
        } else {
            // Other distributions, not x86_64.
            Some("/usr/lib/".to_owned())
        };
    }

    // Try to build with the static lib if a path was received or chosen
    let search_dir = if let Some(search_dir) = search_dir {
        let path = Path::new(&search_dir);
        let cannonical = fs::canonicalize(&path);
        if path.is_relative() {
            println!(
                "cargo:warning=Provided path {:?} is relative.  Path must be \
                 relative to shaderc-sys crate, possibly not your current \
                 working directory",
                &path
            );
        } else if !path.is_dir() {
            println!("cargo:warning=Provided path {:?} is not a directory", &path);
        }
        if (cannonical.is_err()) && explicit_lib_dir_set {
            println!("cargo:warning={:?}", cannonical.err().unwrap());
            println!(
                "cargo:warning=Provided path {:?} could not be canonicallized",
                &path
            );
            None
        } else {
            cannonical.ok()
        }
    } else {
        None
    };

    if let Some(search_dir) = search_dir {
        let search_dir_str = search_dir.to_string_lossy();

        let combined_lib_path =
            search_dir.join(if target_os == "windows" && target_env == "msvc" {
                SHADERC_STATIC_LIB_FILE_MSVC
            } else {
                SHADERC_STATIC_LIB_FILE
            });

        let dylib_name = format!(
            "{}{}{}",
            consts::DLL_PREFIX,
            SHADERC_SHARED_LIB,
            consts::DLL_SUFFIX
        );
        let dylib_path = search_dir.join(dylib_name);

        if let Some((lib_name, kind)) = {
            if combined_lib_path.exists() {
                Some((SHADERC_STATIC_LIB, "static"))
            } else if dylib_path.exists() {
                Some((SHADERC_SHARED_LIB, "dylib"))
            } else {
                None
            }
        } {
            match (target_os.as_str(), target_env.as_str()) {
                ("linux", _) => {
                    println!("cargo:rustc-link-search=native={}", search_dir_str);

                    // Libraries from the SPIRV-Tools project
                    let spirv_tools_lib = probe_linux_lib_kind("SPIRV-Tools", &search_dir);
                    let spirv_tools_opt_lib = probe_linux_lib_kind("SPIRV-Tools-opt", &search_dir);
                    // Libraries from the Glslang project
                    let spirv_lib = probe_linux_lib_kind("SPIRV", &search_dir);
                    let glslang_lib = probe_linux_lib_kind("glslang", &search_dir);

                    match (spirv_tools_lib, spirv_tools_opt_lib, spirv_lib, glslang_lib) {
                        (Some(spirv_tools), Some(spirv_tools_opt), Some(spirv), Some(glslang)) => {
                            println!("cargo:warning=Found and linking system installed Glslang and SPIRV-Tools libraries.");
                            println!("cargo:rustc-link-lib={}=glslang", glslang);
                            println!("cargo:rustc-link-lib={}=SPIRV", spirv);
                            println!("cargo:rustc-link-lib={}=SPIRV-Tools", spirv_tools);
                            println!("cargo:rustc-link-lib={}=SPIRV-Tools-opt", spirv_tools_opt);
                        }
                        _ => {
                            println!(
                                "cargo:warning=Only shaderc library found.  \
                                 Assuming libraries it depends on are built into \
                                 the shaderc library."
                            );
                        }
                    }
                    println!("cargo:rustc-link-lib={}={}", kind, lib_name);
                    println!("cargo:rustc-link-lib=dylib=stdc++");
                    return;
                }
                ("windows", "msvc") => {
                    println!("cargo:warning=Windows MSVC static builds experimental");
                    println!("cargo:rustc-link-search=native={}", search_dir_str);
                    println!("cargo:rustc-link-lib={}={}", kind, lib_name);
                    return;
                }
                ("windows", "gnu") => {
                    println!("cargo:warning=Windows MinGW static builds experimental");
                    println!("cargo:rustc-link-search=native={}", search_dir_str);
                    println!("cargo:rustc-link-lib={}={}", kind, lib_name);
                    println!("cargo:rustc-link-lib=dylib=stdc++");
                    return;
                }
                ("macos", _) => {
                    println!("cargo:warning=MacOS static builds experimental");
                    println!("cargo:rustc-link-search=native={}", search_dir_str);
                    println!("cargo:rustc-link-lib={}={}", kind, lib_name);
                    println!("cargo:rustc-link-lib=dylib=c++");
                    return;
                }
                ("ios", _) => {
                    println!("cargo:warning=MacOS static builds experimental");
                    println!("cargo:rustc-link-search=native={}", search_dir_str);
                    println!("cargo:rustc-link-lib={}={}", kind, lib_name);
                    println!("cargo:rustc-link-lib=dylib=c++");
                    return;
                }
                (_, _) => {
                    println!("cargo:warning=Platform unsupported for linking against system installed shaderc libraries");
                }
            }
        }
    }

    if config_build_from_source {
        println!("cargo:warning=Requested to build from source");
    } else {
        println!(
            "cargo:warning=System installed library not found.  Falling back \
             to build from source"
        );
    }

    let mut finder = cmd_finder::CommandFinder::new();

    finder.must_have("cmake");
    finder.must_have("git");
    finder.must_have("python");

    let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
    let shaderc_dir = Path::new(&manifest_dir).join("build");

    let mut lib_path = if target_env == "msvc" {
        finder.must_have("ninja");
        build_shaderc_msvc(&shaderc_dir)
    } else {
        let has_ninja = finder.maybe_have("ninja").is_some();
        build_shaderc(&shaderc_dir, has_ninja, target_os)
    };

    lib_path.push("lib");
    println!("cargo:rustc-link-search=native={}", lib_path.display());
    println!("cargo:rustc-link-lib=static=shaderc_combined");

    emit_std_cpp_link();
}

fn emit_std_cpp_link() {
    let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap();
    let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap();

    match (target_os.as_str(), target_env.as_str()) {
        ("linux", _) | ("windows", "gnu") => println!("cargo:rustc-link-lib=dylib=stdc++"),
        ("macos", _) => println!("cargo:rustc-link-lib=dylib=c++"),
        _ => {}
    }
}
<commit_msg>Improve cargo build warning messages<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

mod cmd_finder;

use std::env;
use std::env::consts;
use std::fs;
use std::path::{Path, PathBuf};

static SHADERC_STATIC_LIB: &str = "shaderc_combined";
static SHADERC_SHARED_LIB: &str = "shaderc_shared";
static SHADERC_STATIC_LIB_FILE: &str = "libshaderc_combined.a";
static SHADERC_STATIC_LIB_FILE_MSVC: &str = "shaderc_combined.lib";

fn sdk_path() -> Option<PathBuf> {
    let target = std::env::var("TARGET").unwrap();
    use std::process::Command;

    // tvOS (and the simulator) could be added here in the future.
    let sdk = if target == "x86_64-apple-ios" || target == "i386-apple-ios" {
        "iphonesimulator"
    } else if target == "aarch64-apple-ios"
        || target == "armv7-apple-ios"
        || target == "armv7s-apple-ios"
    {
        "iphoneos"
    } else {
        return None;
    };

    let output = if let Ok(out) = Command::new("xcrun")
        .args(&["--sdk", sdk, "--show-sdk-path"])
        .output()
    {
        out.stdout
    } else {
        return None;
    };
    let prefix_str = std::str::from_utf8(&output).expect("invalid output from `xcrun`");
    Some(PathBuf::from(prefix_str.trim_end().to_string()))
}

fn build_shaderc(shaderc_dir: &PathBuf, use_ninja: bool, target_os: String) -> PathBuf {
    let mut config = cmake::Config::new(shaderc_dir);
    config
        .profile("Release")
        .define("CMAKE_POSITION_INDEPENDENT_CODE", "ON")
        .define("SPIRV_SKIP_EXECUTABLES", "ON")
        .define("SPIRV_WERROR", "OFF")
        .define("SHADERC_SKIP_TESTS", "ON")
        .define("CMAKE_INSTALL_LIBDIR", "lib");
    if use_ninja {
        config.generator("Ninja");
    }

    if target_os == "ios" {
        if let Some(path) = sdk_path() {
            config.define("CMAKE_OSX_SYSROOT", path);
        }
    }

    config.build()
}

fn build_shaderc_msvc(shaderc_dir: &PathBuf) -> PathBuf {
    let linkage = env::var("CARGO_CFG_TARGET_FEATURE").unwrap_or_default();

    let mut config = cmake::Config::new(shaderc_dir);
    config
        .profile("Release")
        .define("CMAKE_POSITION_INDEPENDENT_CODE", "ON")
        .define("SPIRV_SKIP_EXECUTABLES", "ON")
        .define("SPIRV_WERROR", "OFF")
        .define("SHADERC_SKIP_TESTS", "ON")
        .define("CMAKE_INSTALL_LIBDIR", "lib")
        .generator("Ninja");

    // cmake-rs tries to be clever on Windows by injecting several
    // C/C++ flags, which causes problems. So I have to manually
    // define CMAKE_*_FLAGS_* here to suppress that.
    let config = if linkage.contains("crt-static") {
        // statically-linked CRT
        config
            .define("CMAKE_C_FLAGS", " /nologo /EHsc /MT")
            .define("CMAKE_CXX_FLAGS", " /nologo /EHsc /MT")
            .define("CMAKE_C_FLAGS_RELEASE", " /nologo /EHsc /MT")
            .define("CMAKE_CXX_FLAGS_RELEASE", " /nologo /EHsc /MT")
    } else {
        // dynamically-linked CRT
        config
            .define("CMAKE_C_FLAGS", " /nologo /EHsc /MD")
            .define("CMAKE_CXX_FLAGS", " /nologo /EHsc /MD")
            .define("CMAKE_C_FLAGS_RELEASE", " /nologo /EHsc /MD")
            .define("CMAKE_CXX_FLAGS_RELEASE", " /nologo /EHsc /MD")
            // prevent shaderc's cmake script messes with crt flags
            .define("SHADERC_ENABLE_SHARED_CRT", "ON")
    };

    config.build()
}

fn probe_linux_lib_kind(lib_name: &str, search_dir: &PathBuf) -> Option<&'static str> {
    if search_dir.join(&format!("lib{}.so", lib_name)).exists() {
        Some("dylib")
    } else if search_dir.join(&format!("lib{}.a", lib_name)).exists() {
        Some("static")
    } else {
        None
    }
}

fn main() {
    let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap();
    let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap();
    let config_build_from_source = env::var("CARGO_FEATURE_BUILD_FROM_SOURCE").is_ok();
    let explicit_lib_dir_set = env::var("SHADERC_LIB_DIR").is_ok();

    // Initialize explicit libshaderc search directory first
    let mut search_dir = if let Ok(lib_dir) = env::var("SHADERC_LIB_DIR") {
        println!(
            "cargo:warning=shaderc: searching native shaderc libraries in '{}'",
            lib_dir
        );
        Some(lib_dir)
    } else {
        None
    };

    // If no explicit path is set and no explicit request is made to build from
    // source, check known locations before falling back to from-source-build
    if search_dir.is_none() && target_os == "linux" && !config_build_from_source {
        println!(
            "cargo:warning=shaderc: searching for native shaderc libraries on system;  \
             use '--features build-from-source' to force building from source code"
        );

        // https://wiki.ubuntu.com/MultiarchSpec
        // https://wiki.debian.org/Multiarch/Implementation
        let debian_arch = match env::var("CARGO_CFG_TARGET_ARCH").unwrap() {
            arch if arch == "x86" => "i386".to_owned(),
            arch => arch,
        };
        let debian_triple_path = format!("/usr/lib/{}-linux-gnu/", debian_arch);

        search_dir = if Path::new(&debian_triple_path).exists() {
            // Debian, Ubuntu and their derivatives.
            Some(debian_triple_path)
        } else if env::var("CARGO_CFG_TARGET_ARCH").unwrap() == "x86_64"
            && Path::new("/usr/lib64/").exists()
        {
            // Other distributions running on x86_64 usually use this path.
            Some("/usr/lib64/".to_owned())
        } else {
            // Other distributions, not x86_64.
            Some("/usr/lib/".to_owned())
        };
    }

    // Try to build with the static lib if a path was received or chosen
    let search_dir = if let Some(search_dir) = search_dir {
        let path = Path::new(&search_dir);
        let cannonical = fs::canonicalize(&path);
        if path.is_relative() {
            println!(
                "cargo:warning=shaderc: the given search path '{:?}' is relative; \
                 path must be relative to shaderc-sys crate, \
                 likely not your current working directory",
                &path
            );
        } else if !path.is_dir() {
            println!(
                "cargo:warning=shaderc: the given search path '{:?}' is not a directory",
                &path
            );
        }
        if (cannonical.is_err()) && explicit_lib_dir_set {
            println!("cargo:warning=shaderc: {:?}", cannonical.err().unwrap());
            println!(
                "cargo:warning=shaderc: failed to canonicalize the given search path '{:?}'",
                &path
            );
            None
        } else {
            cannonical.ok()
        }
    } else {
        None
    };

    if let Some(search_dir) = search_dir {
        let search_dir_str = search_dir.to_string_lossy();

        let combined_lib_path =
            search_dir.join(if target_os == "windows" && target_env == "msvc" {
                SHADERC_STATIC_LIB_FILE_MSVC
            } else {
                SHADERC_STATIC_LIB_FILE
            });

        let dylib_name = format!(
            "{}{}{}",
            consts::DLL_PREFIX,
            SHADERC_SHARED_LIB,
            consts::DLL_SUFFIX
        );
        let dylib_path = search_dir.join(dylib_name);

        if let Some((lib_name, kind)) = {
            if combined_lib_path.exists() {
                Some((SHADERC_STATIC_LIB, "static"))
            } else if dylib_path.exists() {
                Some((SHADERC_SHARED_LIB, "dylib"))
            } else {
                None
            }
        } {
            match (target_os.as_str(), target_env.as_str()) {
                ("linux", _) => {
                    println!("cargo:rustc-link-search=native={}", search_dir_str);

                    // Libraries from the SPIRV-Tools project
                    let spirv_tools_lib = probe_linux_lib_kind("SPIRV-Tools", &search_dir);
                    let spirv_tools_opt_lib = probe_linux_lib_kind("SPIRV-Tools-opt", &search_dir);
                    // Libraries from the Glslang project
                    let spirv_lib = probe_linux_lib_kind("SPIRV", &search_dir);
                    let glslang_lib = probe_linux_lib_kind("glslang", &search_dir);

                    match (spirv_tools_lib, spirv_tools_opt_lib, spirv_lib, glslang_lib) {
                        (Some(spirv_tools), Some(spirv_tools_opt), Some(spirv), Some(glslang)) => {
                            println!(
                                "cargo:warning=shaderc: found and linking glslang and spirv-tools \
                                 libraries installed on system"
                            );
                            println!("cargo:rustc-link-lib={}=glslang", glslang);
                            println!("cargo:rustc-link-lib={}=SPIRV", spirv);
                            println!("cargo:rustc-link-lib={}=SPIRV-Tools", spirv_tools);
                            println!("cargo:rustc-link-lib={}=SPIRV-Tools-opt", spirv_tools_opt);
                        }
                        _ => {
                            println!(
                                "cargo:warning=shaderc: only found the shaderc library; \
                                 assuming libraries it depends on are built into the shaderc library."
                            );
                        }
                    }
                    println!("cargo:rustc-link-lib={}={}", kind, lib_name);
                    println!("cargo:rustc-link-lib=dylib=stdc++");
                    return;
                }
                ("windows", "msvc") => {
                    println!("cargo:warning=shaderc: Windows MSVC static build is experimental");
                    println!("cargo:rustc-link-search=native={}", search_dir_str);
                    println!("cargo:rustc-link-lib={}={}", kind, lib_name);
                    return;
                }
                ("windows", "gnu") => {
                    println!("cargo:warning=shaderc: Windows MinGW static build is experimental");
                    println!("cargo:rustc-link-search=native={}", search_dir_str);
                    println!("cargo:rustc-link-lib={}={}", kind, lib_name);
                    println!("cargo:rustc-link-lib=dylib=stdc++");
                    return;
                }
                ("macos", _) => {
                    println!("cargo:warning=shaderc: macOS static build is experimental");
                    println!("cargo:rustc-link-search=native={}", search_dir_str);
                    println!("cargo:rustc-link-lib={}={}", kind, lib_name);
                    println!("cargo:rustc-link-lib=dylib=c++");
                    return;
                }
                ("ios", _) => {
                    println!("cargo:warning=shaderc: macOS static build is experimental");
                    println!("cargo:rustc-link-search=native={}", search_dir_str);
                    println!("cargo:rustc-link-lib={}={}", kind, lib_name);
                    println!("cargo:rustc-link-lib=dylib=c++");
                    return;
                }
                (_, _) => {
                    println!(
                        "cargo:warning=shaderc: unsupported platform for linking against \
                         native shaderc libraries installed on system"
                    );
                }
            }
        }
    }

    if config_build_from_source {
        println!("cargo:warning=shaderc: requested to build from source");
    } else {
        println!(
            "cargo:warning=shaderc: cannot find native shaderc library on system; \
             falling back to build from source"
        );
    }

    let mut finder = cmd_finder::CommandFinder::new();

    finder.must_have("cmake");
    finder.must_have("git");
    finder.must_have("python");

    let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
    let shaderc_dir = Path::new(&manifest_dir).join("build");

    let mut lib_path = if target_env == "msvc" {
        finder.must_have("ninja");
        build_shaderc_msvc(&shaderc_dir)
    } else {
        let has_ninja = finder.maybe_have("ninja").is_some();
        build_shaderc(&shaderc_dir, has_ninja, target_os)
    };

    lib_path.push("lib");
    println!("cargo:rustc-link-search=native={}", lib_path.display());
    println!("cargo:rustc-link-lib=static=shaderc_combined");

    emit_std_cpp_link();
}

fn emit_std_cpp_link() {
    let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap();
    let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap();

    match (target_os.as_str(), target_env.as_str()) {
        ("linux", _) | ("windows", "gnu") => println!("cargo:rustc-link-lib=dylib=stdc++"),
        ("macos", _) => println!("cargo:rustc-link-lib=dylib=c++"),
        _ => {}
    }
}
<|endoftext|>


<commit_before>extern crate session_types;
extern crate rand;

use session_types::*;
use std::sync::mpsc::{channel, Receiver};
use std::thread::spawn;
use rand::random;

type Server = Recv<u8, Choose<Send<u8, Eps>, Eps>>;
type Client = Send<u8, Offer<Recv<u8, Eps>, Eps>>;

fn handler(c: Chan<(), Server>) {
    let (c, n) = c.recv();
    match n.checked_add(42) {
        Some(n) => c.sel1().send(n).close(),
        None => c.sel2().close(),
    }
}

fn server(rx: Receiver<Chan<(), Server>>) {
    let mut count = 0;
    loop {
        match borrow_request(&rx) {
            Some(c) => {
                spawn(move || handler(c));
                count += 1;
            },
            None => break,
        }
    }
    println!("Handled {} connections", count);
}

fn client(c: Chan<(), Client>) {
    let n = random();
    match c.send(n).offer() {
        Ok(c) => {
            let (c, n2) = c.recv();
            c.close();
            println!("{} + 42 = {}", n, n2);
        },
        Err(c) => {
            c.close();
            println!("{} + 42 is an overflow :(", n);
        }
    }
}

fn main() {
    let (tx, rx) = channel();
    let mut buf = Vec::new();

    let n: u8 = random();
    println!("Spawning {} clients", n);
    for _ in 0..n {
        let tmp = tx.clone();
        buf.push(spawn(move || client(accept(tmp).unwrap())));
    }
    drop(tx);

    server(rx);
    for t in buf {
        t.join().unwrap();
    }
}
<commit_msg>We can use HasDual to implement session types!<commit_after>extern crate session_types;
extern crate rand;

use session_types::*;
use std::sync::mpsc::{channel, Receiver};
use std::thread::spawn;
use rand::random;

type Server = Recv<u8, Choose<Send<u8, Eps>, Eps>>;
type Client = <Server as HasDual>::Dual;

fn handler(c: Chan<(), Server>) {
    let (c, n) = c.recv();
    match n.checked_add(42) {
        Some(n) => c.sel1().send(n).close(),
        None => c.sel2().close(),
    }
}

fn server(rx: Receiver<Chan<(), Server>>) {
    let mut count = 0;
    loop {
        match borrow_request(&rx) {
            Some(c) => {
                spawn(move || handler(c));
                count += 1;
            },
            None => break,
        }
    }
    println!("Handled {} connections", count);
}

fn client(c: Chan<(), Client>) {
    let n = random();
    match c.send(n).offer() {
        Ok(c) => {
            let (c, n2) = c.recv();
            c.close();
            println!("{} + 42 = {}", n, n2);
        },
        Err(c) => {
            c.close();
            println!("{} + 42 is an overflow :(", n);
        }
    }
}

fn main() {
    let (tx, rx) = channel();
    let mut buf = Vec::new();

    let n: u8 = random();
    println!("Spawning {} clients", n);
    for _ in 0..n {
        let tmp = tx.clone();
        buf.push(spawn(move || client(accept(tmp).unwrap())));
    }
    drop(tx);

    server(rx);
    for t in buf {
        t.join().unwrap();
    }
}
<|endoftext|>


<commit_before>//! Basic digital signal processing (DSP) operations
//!
//! Digital signal processing based on real or complex vectors in time or frequency domain.
//! Vectors are expected to typically have a size which is at least in the order
//! of magnitude of a couple of thousand elements. This crate tries to balance between a clear
//! API and performance in terms of processing speed.
//!
//! Take this example:
//!
//! ```
//! # extern crate num_complex;
//! # extern crate basic_dsp_vector;
//! # use basic_dsp_vector::*;
//! # fn main() {
//! let mut vector1 = vec!(1.0, 2.0).to_real_time_vec();
//! let vector2 = vec!(10.0, 11.0).to_real_time_vec();
//! vector1.add(&vector2).expect("Ignoring error handling in examples");
//! # }
//!
//! ```
//! If `vector2` would be a complex or frequency vector then this won't compile. The type mismatch
//! indicates that a conversation is missing and that this might be a programming mistake. This lib uses
//! the Rust type system to catch such errors.
//!
//! DSP algorithms are often executed in loops. If you work with large vectors you typically try to avoid
//! allocating buffers in every iteration. Preallocating buffers is a common practice to safe a little time
//! with every iteration later on, but also to avoid heap fragmentation. At the same time it's a tedious task
//! to calculate the right buffer sizes for all operations. As an attempt to provide a more convenient solution
//! buffer types exist which don't preallocate, but store temporary memory segments so that they can be reused in the
//! next iteration. Here is an example:
//!
//! ```
//! # use std::f32;
//! # use basic_dsp_vector::*;
//! let vector = vec!(1.0, 0.0, -0.5, 0.8660254, -0.5, -0.8660254).to_complex_time_vec();
//! let mut buffer = SingleBuffer::new();
//! let _ = vector.fft(&mut buffer);
//! ```
//!
//! The vector types don't distinguish between the shapes 1xN or Nx1. This is a difference to other
//! conventions such as in MATLAB or GNU Octave.
//! The reason for this decision is that most operations are only defined if the shape of the
//! vector matches. So it appears to be more practical and clearer to implement the few operations
//! where the arguments can be of different shapes as seperate methods. The methods `mul` and `dot_product`
//! are one example for this.
//!
//! The trait definitions in this lib can look complex and might be overwhelming at the beginning.
//! There is a wide range of DSP vectors, e.g. a slice can be DSP vector, a boxed array can be a DSP vector,
//! a standard vector can be a DSP vector and so on. This lib tries to work with all of that and tries
//! to allow all those different DSP vector types to work together. The price for this flexibility is a more complex
//! trait definition. As a mental model, this is what the traits are specifiying:
//! Whenever you have a complex vector in time domain, it's binary operations will work with all other
//! complex vectors in time domain, but not with real valued vectors or frequency domain vectors.
//! And the type `GenDspVec` serves as wild card at compile time since it defers all checks to run time.

#![cfg_attr(feature = "cargo-clippy", feature(tool_lints))]

extern crate arrayvec;
#[cfg(any(feature = "doc", feature = "use_gpu"))]
extern crate clfft;
#[cfg(feature = "std")]
extern crate crossbeam;
#[cfg(feature = "std")]
#[macro_use]
extern crate lazy_static;
#[cfg(feature = "std")]
extern crate num_cpus;
#[cfg(feature = "std")]
extern crate time;
#[cfg(feature = "std")]
extern crate linreg;
extern crate num_complex;
extern crate num_traits;
#[cfg(any(feature = "doc", feature = "use_gpu"))]
extern crate ocl;
extern crate rustfft;
#[cfg(any(feature = "doc", feature = "use_sse", feature = "use_avx"))]
extern crate simd;
#[macro_use]
mod simd_extensions;
pub mod conv_types;
mod multicore_support;
mod vector_types;
pub mod window_functions;
pub use multicore_support::MultiCoreSettings;
pub use multicore_support::print_calibration;
pub use vector_types::*;
mod gpu_support;
mod inline_vector;
use numbers::*;
use std::ops::Range;

pub mod numbers {
    //! Traits from the `num` crate which are used inside `basic_dsp` and extensions to those traits.
    use gpu_support::{Gpu32, Gpu64, GpuFloat, GpuRegTrait};
    pub use num_complex::Complex;
    use num_traits;
    pub use num_traits::Float;
    pub use num_traits::Num;
    pub use num_traits::One;
    use rustfft;
    #[cfg(any(
        feature = "use_sse",
        feature = "use_avx",
        feature = "use_avx512"
    ))]
    use simd;
    use simd_extensions;
    use simd_extensions::*;
    use std::fmt::Debug;

    /// A trait for a numeric value which at least supports a subset of the operations defined in this crate.
    /// Can be an integer or a floating point number. In order to have support for all operations in this crate
    /// a must implement the `RealNumber`.
    pub trait DspNumber:
        Num
        + Copy
        + Clone
        + Send
        + Sync
        + ToSimd
        + Debug
        + num_traits::Signed
        + num_traits::FromPrimitive
        + rustfft::FFTnum
        + 'static
    {
}
    impl<T> DspNumber for T where
        T: Num
            + Copy
            + Clone
            + Send
            + Sync
            + ToSimd
            + Debug
            + num_traits::Signed
            + num_traits::FromPrimitive
            + rustfft::FFTnum
            + 'static
    {}

    /// Associates a number type with a SIMD register type.
    pub trait ToSimd: Sized + Sync + Send {
        /// Type for the SIMD register on the CPU.
        type RegFallback: SimdGeneric<Self>;
        type RegSse: SimdGeneric<Self>;
        type RegAvx: SimdGeneric<Self>;
        type RegAvx512: SimdGeneric<Self>;
        /// Type for the SIMD register on the GPU. Defaults to an arbitrary type if GPU support is not
        /// compiled in.
        type GpuReg: GpuRegTrait;
    }

    impl ToSimd for f32 {
        type RegFallback = simd_extensions::fallback::f32x4;

        #[cfg(feature = "use_sse")]
        type RegSse = simd::f32x4;
        #[cfg(not(feature = "use_sse"))]
        type RegSse = simd_extensions::fallback::f32x4;

        #[cfg(feature = "use_avx")]
        type RegAvx = simd::x86::avx::f32x8;
        #[cfg(not(feature = "use_avx"))]
        type RegAvx = simd_extensions::fallback::f32x4;

        #[cfg(feature = "use_avx512")]
        type RegAvx512 = stdsimd::simd::f32x16; // Type is missing in SIMD
        #[cfg(not(feature = "use_avx512"))]
        type RegAvx512 = simd_extensions::fallback::f32x4;

        type GpuReg = Gpu32;
    }

    impl ToSimd for f64 {
        type RegFallback = simd_extensions::fallback::f64x2;

        #[cfg(feature = "use_sse")]
        type RegSse = simd::x86::sse2::f64x2;
        #[cfg(not(feature = "use_sse"))]
        type RegSse = simd_extensions::fallback::f64x2;

        #[cfg(feature = "use_avx")]
        type RegAvx = simd::x86::avx::f64x4;
        #[cfg(not(feature = "use_avx"))]
        type RegAvx = simd_extensions::fallback::f64x2;

        #[cfg(feature = "use_avx512")]
        type RegAvx512 = stdsimd::simd::f64x8; // Type is missing in SIMD
        #[cfg(not(feature = "use_avx512"))]
        type RegAvx512 = simd_extensions::fallback::f64x2;

        type GpuReg = Gpu64;
    }

    /// A real floating pointer number intended to abstract over `f32` and `f64`.
    pub trait RealNumber: Float + DspNumber + GpuFloat + num_traits::FloatConst {}
    impl<T> RealNumber for T where T: Float + DspNumber + GpuFloat + num_traits::FloatConst {}

    /// This trait is necessary so that we can define zero for types outside this crate.
    /// It calls the `num_traits::Zero` trait where possible.
    pub trait Zero {
        fn zero() -> Self;
    }

    impl<T> Zero for T
    where
        T: DspNumber,
    {
        fn zero() -> Self {
            <Self as num_traits::Zero>::zero()
        }
    }

    impl<T> Zero for Complex<T>
    where
        T: DspNumber,
    {
        fn zero() -> Self {
            <Self as num_traits::Zero>::zero()
        }
    }
}

/// Transmutes a slice. Both S and D must be `#[repr(C)]`.
/// The code panics if the slice has a length which doesn't allow conversion.
fn transmute_slice<S, D>(source: &[S]) -> &[D] {
    let len = get_target_slice_len::<S, D>(source);
    unsafe {
        let trans: &[D] = &*(source as *const [S] as *const [D]);
        std::slice::from_raw_parts(trans.as_ptr(), len)
    }
}

/// Transmutes a mutable slice. Both S and D must be `#[repr(C)]`.
/// The code panics if the slice has a length which doesn't allow conversion.
fn transmute_slice_mut<S, D>(source: &mut [S]) -> &mut[D] {
    let len = get_target_slice_len::<S, D>(source);
    unsafe {
        let trans: &mut [D] = &mut *(source as *mut [S] as *mut [D]);
        std::slice::from_raw_parts_mut(trans.as_mut_ptr(), len)
    }
}

/// Helper method which finds the length of a target slice and also perform checks on the length
fn get_target_slice_len<S, D>(source: &[S]) -> usize {
    let to_larger_type = std::mem::size_of::<D>() >= std::mem::size_of::<S>();
    if to_larger_type {
        assert_eq!(std::mem::size_of::<D>() % std::mem::size_of::<S>(), 0);
        let ratio = std::mem::size_of::<D>() / std::mem::size_of::<S>();
        assert_eq!(source.len() % ratio, 0);
        source.len() / ratio
    } else {
        assert_eq!(std::mem::size_of::<S>() % std::mem::size_of::<D>(), 0);
        let ratio = std::mem::size_of::<S>() / std::mem::size_of::<D>();
        source.len() * ratio
    }
}

// Returns a complex slice from a real slice
fn array_to_complex<T>(array: &[T]) -> &[Complex<T>] {
    transmute_slice(array)
}

// Returns a complex slice from a real slice
fn array_to_complex_mut<T>(array: &mut [T]) -> &mut [Complex<T>] {
    transmute_slice_mut(array)
}

/// Copies memory inside a slice
fn memcpy<T: Copy>(data: &mut [T], from: Range<usize>, to: usize) {
    use std::ptr::copy;
    assert!(from.start <= from.end);
    assert!(from.end <= data.len());
    assert!(to <= data.len() - (from.end - from.start));
    unsafe {
        let ptr = data.as_mut_ptr();
        copy(
            ptr.add(from.start),
            ptr.add(to),
            from.end - from.start,
        )
    }
}

// Zeros a range within the slice
fn memzero<T: Copy>(data: &mut [T], range: Range<usize>) {
    use std::ptr::write_bytes;
    assert!(range.start <= range.end);
    assert!(range.end <= data.len());
    unsafe {
        let ptr = data.as_mut_ptr();
        write_bytes(ptr.add(range.start), 0, range.end - range.start);
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use simd_extensions::Simd;

    #[test]
    fn to_simd_test() {
        // This is more a check for syntax. So if it compiles
        // then the test already passes. The assert is then only
        // a sanity check.
        let reg = <f32 as ToSimd>::RegFallback::splat(1.0);
        let sum = reg.sum_real();
        assert!(sum > 0.0);
    }
}
<commit_msg>Ignoring a LINT after fixing it<commit_after>//! Basic digital signal processing (DSP) operations
//!
//! Digital signal processing based on real or complex vectors in time or frequency domain.
//! Vectors are expected to typically have a size which is at least in the order
//! of magnitude of a couple of thousand elements. This crate tries to balance between a clear
//! API and performance in terms of processing speed.
//!
//! Take this example:
//!
//! ```
//! # extern crate num_complex;
//! # extern crate basic_dsp_vector;
//! # use basic_dsp_vector::*;
//! # fn main() {
//! let mut vector1 = vec!(1.0, 2.0).to_real_time_vec();
//! let vector2 = vec!(10.0, 11.0).to_real_time_vec();
//! vector1.add(&vector2).expect("Ignoring error handling in examples");
//! # }
//!
//! ```
//! If `vector2` would be a complex or frequency vector then this won't compile. The type mismatch
//! indicates that a conversation is missing and that this might be a programming mistake. This lib uses
//! the Rust type system to catch such errors.
//!
//! DSP algorithms are often executed in loops. If you work with large vectors you typically try to avoid
//! allocating buffers in every iteration. Preallocating buffers is a common practice to safe a little time
//! with every iteration later on, but also to avoid heap fragmentation. At the same time it's a tedious task
//! to calculate the right buffer sizes for all operations. As an attempt to provide a more convenient solution
//! buffer types exist which don't preallocate, but store temporary memory segments so that they can be reused in the
//! next iteration. Here is an example:
//!
//! ```
//! # use std::f32;
//! # use basic_dsp_vector::*;
//! let vector = vec!(1.0, 0.0, -0.5, 0.8660254, -0.5, -0.8660254).to_complex_time_vec();
//! let mut buffer = SingleBuffer::new();
//! let _ = vector.fft(&mut buffer);
//! ```
//!
//! The vector types don't distinguish between the shapes 1xN or Nx1. This is a difference to other
//! conventions such as in MATLAB or GNU Octave.
//! The reason for this decision is that most operations are only defined if the shape of the
//! vector matches. So it appears to be more practical and clearer to implement the few operations
//! where the arguments can be of different shapes as seperate methods. The methods `mul` and `dot_product`
//! are one example for this.
//!
//! The trait definitions in this lib can look complex and might be overwhelming at the beginning.
//! There is a wide range of DSP vectors, e.g. a slice can be DSP vector, a boxed array can be a DSP vector,
//! a standard vector can be a DSP vector and so on. This lib tries to work with all of that and tries
//! to allow all those different DSP vector types to work together. The price for this flexibility is a more complex
//! trait definition. As a mental model, this is what the traits are specifiying:
//! Whenever you have a complex vector in time domain, it's binary operations will work with all other
//! complex vectors in time domain, but not with real valued vectors or frequency domain vectors.
//! And the type `GenDspVec` serves as wild card at compile time since it defers all checks to run time.

#![cfg_attr(feature = "cargo-clippy", feature(tool_lints))]
#![cfg_attr(feature = "cargo-clippy", allow(clippy::new_without_default_derive))] // This LINT gives false positives

extern crate arrayvec;
#[cfg(any(feature = "doc", feature = "use_gpu"))]
extern crate clfft;
#[cfg(feature = "std")]
extern crate crossbeam;
#[cfg(feature = "std")]
#[macro_use]
extern crate lazy_static;
#[cfg(feature = "std")]
extern crate num_cpus;
#[cfg(feature = "std")]
extern crate time;
#[cfg(feature = "std")]
extern crate linreg;
extern crate num_complex;
extern crate num_traits;
#[cfg(any(feature = "doc", feature = "use_gpu"))]
extern crate ocl;
extern crate rustfft;
#[cfg(any(feature = "doc", feature = "use_sse", feature = "use_avx"))]
extern crate simd;
#[macro_use]
mod simd_extensions;
pub mod conv_types;
mod multicore_support;
mod vector_types;
pub mod window_functions;
pub use multicore_support::MultiCoreSettings;
pub use multicore_support::print_calibration;
pub use vector_types::*;
mod gpu_support;
mod inline_vector;
use numbers::*;
use std::ops::Range;

pub mod numbers {
    //! Traits from the `num` crate which are used inside `basic_dsp` and extensions to those traits.
    use gpu_support::{Gpu32, Gpu64, GpuFloat, GpuRegTrait};
    pub use num_complex::Complex;
    use num_traits;
    pub use num_traits::Float;
    pub use num_traits::Num;
    pub use num_traits::One;
    use rustfft;
    #[cfg(any(
        feature = "use_sse",
        feature = "use_avx",
        feature = "use_avx512"
    ))]
    use simd;
    use simd_extensions;
    use simd_extensions::*;
    use std::fmt::Debug;

    /// A trait for a numeric value which at least supports a subset of the operations defined in this crate.
    /// Can be an integer or a floating point number. In order to have support for all operations in this crate
    /// a must implement the `RealNumber`.
    pub trait DspNumber:
        Num
        + Copy
        + Clone
        + Send
        + Sync
        + ToSimd
        + Debug
        + num_traits::Signed
        + num_traits::FromPrimitive
        + rustfft::FFTnum
        + 'static
    {
}
    impl<T> DspNumber for T where
        T: Num
            + Copy
            + Clone
            + Send
            + Sync
            + ToSimd
            + Debug
            + num_traits::Signed
            + num_traits::FromPrimitive
            + rustfft::FFTnum
            + 'static
    {}

    /// Associates a number type with a SIMD register type.
    pub trait ToSimd: Sized + Sync + Send {
        /// Type for the SIMD register on the CPU.
        type RegFallback: SimdGeneric<Self>;
        type RegSse: SimdGeneric<Self>;
        type RegAvx: SimdGeneric<Self>;
        type RegAvx512: SimdGeneric<Self>;
        /// Type for the SIMD register on the GPU. Defaults to an arbitrary type if GPU support is not
        /// compiled in.
        type GpuReg: GpuRegTrait;
    }

    impl ToSimd for f32 {
        type RegFallback = simd_extensions::fallback::f32x4;

        #[cfg(feature = "use_sse")]
        type RegSse = simd::f32x4;
        #[cfg(not(feature = "use_sse"))]
        type RegSse = simd_extensions::fallback::f32x4;

        #[cfg(feature = "use_avx")]
        type RegAvx = simd::x86::avx::f32x8;
        #[cfg(not(feature = "use_avx"))]
        type RegAvx = simd_extensions::fallback::f32x4;

        #[cfg(feature = "use_avx512")]
        type RegAvx512 = stdsimd::simd::f32x16; // Type is missing in SIMD
        #[cfg(not(feature = "use_avx512"))]
        type RegAvx512 = simd_extensions::fallback::f32x4;

        type GpuReg = Gpu32;
    }

    impl ToSimd for f64 {
        type RegFallback = simd_extensions::fallback::f64x2;

        #[cfg(feature = "use_sse")]
        type RegSse = simd::x86::sse2::f64x2;
        #[cfg(not(feature = "use_sse"))]
        type RegSse = simd_extensions::fallback::f64x2;

        #[cfg(feature = "use_avx")]
        type RegAvx = simd::x86::avx::f64x4;
        #[cfg(not(feature = "use_avx"))]
        type RegAvx = simd_extensions::fallback::f64x2;

        #[cfg(feature = "use_avx512")]
        type RegAvx512 = stdsimd::simd::f64x8; // Type is missing in SIMD
        #[cfg(not(feature = "use_avx512"))]
        type RegAvx512 = simd_extensions::fallback::f64x2;

        type GpuReg = Gpu64;
    }

    /// A real floating pointer number intended to abstract over `f32` and `f64`.
    pub trait RealNumber: Float + DspNumber + GpuFloat + num_traits::FloatConst {}
    impl<T> RealNumber for T where T: Float + DspNumber + GpuFloat + num_traits::FloatConst {}

    /// This trait is necessary so that we can define zero for types outside this crate.
    /// It calls the `num_traits::Zero` trait where possible.
    pub trait Zero {
        fn zero() -> Self;
    }

    impl<T> Zero for T
    where
        T: DspNumber,
    {
        fn zero() -> Self {
            <Self as num_traits::Zero>::zero()
        }
    }

    impl<T> Zero for Complex<T>
    where
        T: DspNumber,
    {
        fn zero() -> Self {
            <Self as num_traits::Zero>::zero()
        }
    }
}

/// Transmutes a slice. Both S and D must be `#[repr(C)]`.
/// The code panics if the slice has a length which doesn't allow conversion.
fn transmute_slice<S, D>(source: &[S]) -> &[D] {
    let len = get_target_slice_len::<S, D>(source);
    unsafe {
        let trans: &[D] = &*(source as *const [S] as *const [D]);
        std::slice::from_raw_parts(trans.as_ptr(), len)
    }
}

/// Transmutes a mutable slice. Both S and D must be `#[repr(C)]`.
/// The code panics if the slice has a length which doesn't allow conversion.
fn transmute_slice_mut<S, D>(source: &mut [S]) -> &mut[D] {
    let len = get_target_slice_len::<S, D>(source);
    unsafe {
        let trans: &mut [D] = &mut *(source as *mut [S] as *mut [D]);
        std::slice::from_raw_parts_mut(trans.as_mut_ptr(), len)
    }
}

/// Helper method which finds the length of a target slice and also perform checks on the length
fn get_target_slice_len<S, D>(source: &[S]) -> usize {
    let to_larger_type = std::mem::size_of::<D>() >= std::mem::size_of::<S>();
    if to_larger_type {
        assert_eq!(std::mem::size_of::<D>() % std::mem::size_of::<S>(), 0);
        let ratio = std::mem::size_of::<D>() / std::mem::size_of::<S>();
        assert_eq!(source.len() % ratio, 0);
        source.len() / ratio
    } else {
        assert_eq!(std::mem::size_of::<S>() % std::mem::size_of::<D>(), 0);
        let ratio = std::mem::size_of::<S>() / std::mem::size_of::<D>();
        source.len() * ratio
    }
}

// Returns a complex slice from a real slice
fn array_to_complex<T>(array: &[T]) -> &[Complex<T>] {
    transmute_slice(array)
}

// Returns a complex slice from a real slice
fn array_to_complex_mut<T>(array: &mut [T]) -> &mut [Complex<T>] {
    transmute_slice_mut(array)
}

/// Copies memory inside a slice
fn memcpy<T: Copy>(data: &mut [T], from: Range<usize>, to: usize) {
    use std::ptr::copy;
    assert!(from.start <= from.end);
    assert!(from.end <= data.len());
    assert!(to <= data.len() - (from.end - from.start));
    unsafe {
        let ptr = data.as_mut_ptr();
        copy(
            ptr.add(from.start),
            ptr.add(to),
            from.end - from.start,
        )
    }
}

// Zeros a range within the slice
fn memzero<T: Copy>(data: &mut [T], range: Range<usize>) {
    use std::ptr::write_bytes;
    assert!(range.start <= range.end);
    assert!(range.end <= data.len());
    unsafe {
        let ptr = data.as_mut_ptr();
        write_bytes(ptr.add(range.start), 0, range.end - range.start);
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use simd_extensions::Simd;

    #[test]
    fn to_simd_test() {
        // This is more a check for syntax. So if it compiles
        // then the test already passes. The assert is then only
        // a sanity check.
        let reg = <f32 as ToSimd>::RegFallback::splat(1.0);
        let sum = reg.sum_real();
        assert!(sum > 0.0);
    }
}
<|endoftext|>