<commit_before>use std::collections::HashMap;
use std::io;
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::Duration;

use abstract_ns::HostResolve;
use futures::Stream;
use futures::future::{Future};
use futures::sync::oneshot::{channel as oneshot, Sender, Receiver};
use futures_cpupool;
use ns_router::{self, SubscribeExt};
use ns_std_threaded;
use self_meter_http::Meter;
use tk_http::server::Proto;
use tk_http;
use tk_listen::ListenExt;
use tokio_core::net::TcpListener;
use tokio_core::reactor::{Handle};

use config::{ListenSocket, ConfigCell};
use incoming::Router;
use chat;
use runtime::Runtime;
use http_pools::{HttpPools};
use handlers::files::{DiskPools};
use request_id;

pub struct State {
    http_pools: HttpPools,
    session_pools: chat::SessionPools,
    disk_pools: DiskPools,
    ns: ns_router::Router,
    listener_shutters: HashMap<SocketAddr, Sender<()>>,
    replication_session: chat::ReplicationSession,
    runtime: Arc<Runtime>,
}

pub fn spawn_listener(addr: SocketAddr, handle: &Handle,
    runtime: &Arc<Runtime>, shutter: Receiver<()>)
    -> Result<(), io::Error>
{
    let root = runtime.config.get();
    let r1 = runtime.clone();
    let r2 = runtime.clone();
    let listener = TcpListener::bind(&addr, &handle)?;
    // TODO(tailhook) how to update?
    let hcfg = tk_http::server::Config::new()
        .inflight_request_limit(root.pipeline_depth)
        // TODO(tailhook) make it configurable?
        .inflight_request_prealoc(0)
        .first_byte_timeout(root.first_byte_timeout)
        .keep_alive_timeout(root.keep_alive_timeout)
        .headers_timeout(root.headers_timeout)
        .input_body_byte_timeout(root.input_body_byte_timeout)
        .input_body_whole_timeout(root.input_body_whole_timeout)
        .output_body_byte_timeout(root.output_body_byte_timeout)
        .output_body_whole_timeout(root.output_body_whole_timeout)
        .done();
    let h1 = handle.clone();

    handle.spawn(
        listener.incoming()
        .sleep_on_error(r1.config.get().listen_error_timeout, &r1.handle)
        .map(move |(socket, saddr)| {
             Proto::new(socket, &hcfg,
                Router::new(saddr, r2.clone(), h1.clone()), &h1)
             .map_err(|e| debug!("Http protocol error: {}", e))
        })
        .listen(root.max_connections)
        .select(shutter.map_err(|_| unreachable!()))
        .map(move |(_, _)| info!("Listener {} exited", addr))
        .map_err(move |(_, _)| info!("Listener {} exited", addr))
    );
    Ok(())
}

pub fn populate_loop(handle: &Handle, cfg: &ConfigCell, verbose: bool)
    -> State
{
    let meter = Meter::new();
    meter.spawn_scanner(handle);
    meter.track_current_thread_by_name();

    let ns_pool = {
        let m1 = meter.clone();
        let m2 = meter.clone();
        futures_cpupool::Builder::new()
        // TODO(tailhook) configure it
        .pool_size(5)
        .name_prefix("ns-resolver-")
        .after_start(move || m1.track_current_thread_by_name())
        .before_stop(move || m2.untrack_current_thread())
        .create()
    };

    let std = ns_std_threaded::ThreadedResolver::use_pool(ns_pool);

    // TODO(tailhook) add config for it, allow update
    let resolver = ns_router::Router::from_config(
        &ns_router::Config::new()
        .set_fallthrough(std.null_service_resolver()
            // TODO(tailhook) allow configure interval
            .interval_subscriber(Duration::new(1, 0), handle))
        .done(),
        handle);

    let server_id = request_id::new();
    let http_pools = HttpPools::new();
    let processor = chat::Processor::new();
    let mut replication_session = chat::ReplicationSession::new(
        processor.clone(), &resolver, handle, &server_id);
    let session_pools = chat::SessionPools::new(
        processor, replication_session.remote_sender.clone());
    let disk_pools = DiskPools::new(&meter);
    let runtime = Arc::new(Runtime {
        config: cfg.clone(),
        handle: handle.clone(),
        http_pools: http_pools.clone(),
        session_pools: session_pools.clone(),
        disk_pools: disk_pools.clone(),
        meter: meter,
        server_id: server_id,
    });
    let root = cfg.get();

    let mut listener_shutters = HashMap::new();

    // TODO(tailhook) do something when config updates
    for sock in &root.listen {
        match sock {
            &ListenSocket::Tcp(addr) => {
                if verbose {
                    println!("Listening at {}", addr);
                }
                let (tx, rx) = oneshot();
                // TODO(tailhook) wait and retry on error
                match spawn_listener(addr, handle, &runtime, rx) {
                    Ok(()) => {
                        listener_shutters.insert(addr, tx);
                    }
                    Err(e) => {
                        error!("Error listening {}: {}. Will retry on next \
                                configuration reload", addr, e);
                    }
                }
            }
        }
    }

    disk_pools.update(&root.disk_pools);
    http_pools.update(&root.http_destinations, &resolver, handle);
    session_pools.update(&root.session_pools, handle, &runtime);
    replication_session.update(&root.replication, handle, &runtime);
    State {
        ns: resolver,
        http_pools: http_pools,
        session_pools: session_pools,
        replication_session: replication_session,
        listener_shutters: listener_shutters,
        runtime: runtime,
        disk_pools: disk_pools,
    }
}

#[allow(dead_code)]
pub fn update_loop(state: &mut State, cfg: &ConfigCell, handle: &Handle) {
    // TODO(tailhook) update listening sockets
    state.disk_pools.update(&cfg.get().disk_pools);
    state.http_pools.update(&cfg.get().http_destinations, &state.ns, handle);
    state.session_pools.update(&cfg.get().session_pools,
        handle, &state.runtime);
    state.replication_session.update(&cfg.get().replication,
        handle, &state.runtime);
}
<commit_msg>Add a log message on server startup<commit_after>use std::collections::HashMap;
use std::io;
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::Duration;

use abstract_ns::HostResolve;
use futures::Stream;
use futures::future::{Future};
use futures::sync::oneshot::{channel as oneshot, Sender, Receiver};
use futures_cpupool;
use ns_router::{self, SubscribeExt};
use ns_std_threaded;
use self_meter_http::Meter;
use tk_http::server::Proto;
use tk_http;
use tk_listen::ListenExt;
use tokio_core::net::TcpListener;
use tokio_core::reactor::{Handle};

use config::{ListenSocket, ConfigCell};
use incoming::Router;
use chat;
use runtime::Runtime;
use http_pools::{HttpPools};
use handlers::files::{DiskPools};
use request_id;

pub struct State {
    http_pools: HttpPools,
    session_pools: chat::SessionPools,
    disk_pools: DiskPools,
    ns: ns_router::Router,
    listener_shutters: HashMap<SocketAddr, Sender<()>>,
    replication_session: chat::ReplicationSession,
    runtime: Arc<Runtime>,
}

pub fn spawn_listener(addr: SocketAddr, handle: &Handle,
    runtime: &Arc<Runtime>, shutter: Receiver<()>)
    -> Result<(), io::Error>
{
    let root = runtime.config.get();
    let r1 = runtime.clone();
    let r2 = runtime.clone();
    let listener = TcpListener::bind(&addr, &handle)?;
    // TODO(tailhook) how to update?
    let hcfg = tk_http::server::Config::new()
        .inflight_request_limit(root.pipeline_depth)
        // TODO(tailhook) make it configurable?
        .inflight_request_prealoc(0)
        .first_byte_timeout(root.first_byte_timeout)
        .keep_alive_timeout(root.keep_alive_timeout)
        .headers_timeout(root.headers_timeout)
        .input_body_byte_timeout(root.input_body_byte_timeout)
        .input_body_whole_timeout(root.input_body_whole_timeout)
        .output_body_byte_timeout(root.output_body_byte_timeout)
        .output_body_whole_timeout(root.output_body_whole_timeout)
        .done();
    let h1 = handle.clone();

    handle.spawn(
        listener.incoming()
        .sleep_on_error(r1.config.get().listen_error_timeout, &r1.handle)
        .map(move |(socket, saddr)| {
             Proto::new(socket, &hcfg,
                Router::new(saddr, r2.clone(), h1.clone()), &h1)
             .map_err(|e| debug!("Http protocol error: {}", e))
        })
        .listen(root.max_connections)
        .select(shutter.map_err(|_| unreachable!()))
        .map(move |(_, _)| info!("Listener {} exited", addr))
        .map_err(move |(_, _)| info!("Listener {} exited", addr))
    );
    Ok(())
}

pub fn populate_loop(handle: &Handle, cfg: &ConfigCell, verbose: bool)
    -> State
{
    let meter = Meter::new();
    meter.spawn_scanner(handle);
    meter.track_current_thread_by_name();

    let ns_pool = {
        let m1 = meter.clone();
        let m2 = meter.clone();
        futures_cpupool::Builder::new()
        // TODO(tailhook) configure it
        .pool_size(5)
        .name_prefix("ns-resolver-")
        .after_start(move || m1.track_current_thread_by_name())
        .before_stop(move || m2.untrack_current_thread())
        .create()
    };

    let std = ns_std_threaded::ThreadedResolver::use_pool(ns_pool);

    // TODO(tailhook) add config for it, allow update
    let resolver = ns_router::Router::from_config(
        &ns_router::Config::new()
        .set_fallthrough(std.null_service_resolver()
            // TODO(tailhook) allow configure interval
            .interval_subscriber(Duration::new(1, 0), handle))
        .done(),
        handle);

    let server_id = request_id::new();
    let http_pools = HttpPools::new();
    let processor = chat::Processor::new();
    let mut replication_session = chat::ReplicationSession::new(
        processor.clone(), &resolver, handle, &server_id);
    let session_pools = chat::SessionPools::new(
        processor, replication_session.remote_sender.clone());
    let disk_pools = DiskPools::new(&meter);
    let runtime = Arc::new(Runtime {
        config: cfg.clone(),
        handle: handle.clone(),
        http_pools: http_pools.clone(),
        session_pools: session_pools.clone(),
        disk_pools: disk_pools.clone(),
        meter: meter,
        server_id: server_id,
    });
    let root = cfg.get();

    warn!("Started with server_id {}", server_id);

    let mut listener_shutters = HashMap::new();

    // TODO(tailhook) do something when config updates
    for sock in &root.listen {
        match sock {
            &ListenSocket::Tcp(addr) => {
                if verbose {
                    println!("Listening at {}", addr);
                }
                let (tx, rx) = oneshot();
                // TODO(tailhook) wait and retry on error
                match spawn_listener(addr, handle, &runtime, rx) {
                    Ok(()) => {
                        listener_shutters.insert(addr, tx);
                    }
                    Err(e) => {
                        error!("Error listening {}: {}. Will retry on next \
                                configuration reload", addr, e);
                    }
                }
            }
        }
    }

    disk_pools.update(&root.disk_pools);
    http_pools.update(&root.http_destinations, &resolver, handle);
    session_pools.update(&root.session_pools, handle, &runtime);
    replication_session.update(&root.replication, handle, &runtime);
    State {
        ns: resolver,
        http_pools: http_pools,
        session_pools: session_pools,
        replication_session: replication_session,
        listener_shutters: listener_shutters,
        runtime: runtime,
        disk_pools: disk_pools,
    }
}

#[allow(dead_code)]
pub fn update_loop(state: &mut State, cfg: &ConfigCell, handle: &Handle) {
    // TODO(tailhook) update listening sockets
    state.disk_pools.update(&cfg.get().disk_pools);
    state.http_pools.update(&cfg.get().http_destinations, &state.ns, handle);
    state.session_pools.update(&cfg.get().session_pools,
        handle, &state.runtime);
    state.replication_session.update(&cfg.get().replication,
        handle, &state.runtime);
}
<|endoftext|>


<commit_before>//! Various APRS data types
//!
//! Data is broken down into objects depending on its purpose. See each struct definition for its
//! description.

use constants;
use functions;
use geo_util;

pub struct Frame {
    pub source: Callsign,
    pub destination: Callsign,
    pub path: Vec<Callsign>,
    pub info: InformationField,
}

impl Frame {
    pub fn new() -> Frame {
        Frame {
            source: Callsign::new(),
            destination: Callsign::new(),
            path: Vec::new(),
            info: InformationField::new()
        }
    }
}

impl Frame {
    pub fn set_source(&mut self, source: &[u8]) {
        self.source = functions::parse_callsign(source);
    }
    pub fn set_destination(&mut self, dest: &[u8]) {
        self.destination = functions::parse_callsign(dest);
    }
    pub fn set_path(&mut self, path: Vec<Vec<u8>>) {
        for pth in path {
            let byts: &[u8] = pth.as_slice();
            self.path.push(functions::parse_callsign(byts));
        }
    }
    pub fn set_info(&mut self, info: &[u8]) {
        self.info = functions::parse_info_field(info);
    }
    pub fn encode_ax25(&self) -> Vec<u8> {
        // TODO: find a way to return raw bytes
        let mut encoded_frame: Vec<u8> = Vec::new();
        encoded_frame.push(constants::AX25_FLAG);
        encoded_frame.extend(self.destination.encode_ax25());
        encoded_frame.extend(self.source.encode_ax25());
        for path_call in self.path.iter() {
            encoded_frame.extend(path_call.encode_ax25());
        }
        encoded_frame.extend(self.info.data.as_slice());
        encoded_frame
    }
}

pub struct PositionFrame {
    pub source: Callsign,
    pub destination: Callsign,
    pub path: Vec<Callsign>,
    pub info: InformationField,
    pub comment: Vec<u8>,
    pub table: Vec<u8>,
    pub symbol: u8,
    pub lat: f32,
    pub lng: f32,
    pub ambiguity: usize
}

impl PositionFrame {
    pub fn new() -> PositionFrame {
        PositionFrame {
            source: Callsign::new(),
            destination: Callsign::new(),
            path: Vec::new(),
            info: InformationField::new(),
            comment: Vec::new(),
            table: Vec::new(),
            symbol: 0,
            lat: 0.0,
            lng: 0.0,
            ambiguity: 0
        }
    }
}

impl PositionFrame {
    pub fn set_source(&mut self, source: &[u8]) {
        self.source = functions::parse_callsign(source);
    }
    pub fn set_destination(&mut self, dest: &[u8]) {
        self.destination = functions::parse_callsign(dest);
    }
    pub fn set_path(&mut self, path: Vec<Vec<u8>>) {
        for pth in path {
            let byts: &[u8] = pth.as_slice();
            self.path.push(functions::parse_callsign(byts));
        }
    }
    pub fn create_info_field(&self) -> Vec<u8> {
        let lat = geo_util::dec2dm_lat(self.lat);
        let lat_enc = geo_util::ambiguate(&lat, self.ambiguity);
        let lng = geo_util::dec2dm_lng(self.lng);
        let lng_enc = geo_util::ambiguate(&lng, self.ambiguity);

        let mut frame: Vec<u8> = Vec::new();
        frame.push(b'=');
        frame.extend(lat_enc.as_bytes());
        frame.extend(&self.table);
        frame.extend(lng_enc.as_bytes());
        frame.push(self.symbol);
        frame.extend(&self.comment);

        frame
    }
}

pub struct Callsign {
    pub callsign: String,
    pub ssid: u8,
    pub digi: bool,
}

impl Callsign {
    pub fn new() -> Callsign {
        Callsign {
            callsign: String::new(),
            ssid: 0,
            digi: false,
        }
    }
}

impl Callsign {
    pub fn set_callsign(&mut self, callsign: String) {
        self.callsign = callsign;
        // self.callsign = String::from_utf8(callsign.to_vec()).unwrap();
    }
    pub fn set_ssid(&mut self, ssid: u8) {
        self.ssid = ssid;
    }
    pub fn set_digi(&mut self, digi: bool) {
        self.digi = digi;
    }
    pub fn encode_ax25(&self) -> Vec<u8> {
        // TODO: find a way to return raw bytes
        let mut encoded_callsign: Vec<u8> = Vec::new();
        let mut encoded_ssid = (self.ssid << 1) | 0x60;
        let mut _callsign: String = String::new();

        if self.digi {
            encoded_ssid = encoded_ssid | 0x80;
        }
        for chr in self.callsign.chars() {
            _callsign.push(chr);
        }
        while _callsign.len() < 6 {
            _callsign.push(' ');
        }
        for byt in _callsign.bytes() {
            encoded_callsign.push(byt << 1);
        }
        encoded_callsign.push(encoded_ssid);

        encoded_callsign
    }
}

pub struct InformationField {
    pub data: Vec<u8>,
    pub data_type: String,
    pub safe: bool,
}

impl InformationField {
    pub fn new() -> InformationField {
        InformationField {
            data: Vec::new(),
            data_type: String::new(),
            safe: false
        }
    }
}

impl InformationField {
    pub fn set_data(&mut self, raw_data: &[u8]) {
        self.data = raw_data.to_vec();
    }
    pub fn set_data_type(&mut self, data_type: &str) {
        self.data_type = data_type.to_string();
    }
    pub fn set_safe(&mut self, safe: bool) {
        self.safe = safe;
    }
}

pub struct APRS;

pub struct TCP;

pub struct UDP;

pub struct HTTP;
<commit_msg>Forgot method to set info (most useful after instantiation).<commit_after>//! Various APRS data types
//!
//! Data is broken down into objects depending on its purpose. See each struct definition for its
//! description.

use constants;
use functions;
use geo_util;

pub struct Frame {
    pub source: Callsign,
    pub destination: Callsign,
    pub path: Vec<Callsign>,
    pub info: InformationField,
}

impl Frame {
    pub fn new() -> Frame {
        Frame {
            source: Callsign::new(),
            destination: Callsign::new(),
            path: Vec::new(),
            info: InformationField::new()
        }
    }
}

impl Frame {
    pub fn set_source(&mut self, source: &[u8]) {
        self.source = functions::parse_callsign(source);
    }
    pub fn set_destination(&mut self, dest: &[u8]) {
        self.destination = functions::parse_callsign(dest);
    }
    pub fn set_path(&mut self, path: Vec<Vec<u8>>) {
        for pth in path {
            let byts: &[u8] = pth.as_slice();
            self.path.push(functions::parse_callsign(byts));
        }
    }
    pub fn set_info(&mut self, info: &[u8]) {
        self.info = functions::parse_info_field(info);
    }
    pub fn encode_ax25(&self) -> Vec<u8> {
        // TODO: find a way to return raw bytes
        let mut encoded_frame: Vec<u8> = Vec::new();
        encoded_frame.push(constants::AX25_FLAG);
        encoded_frame.extend(self.destination.encode_ax25());
        encoded_frame.extend(self.source.encode_ax25());
        for path_call in self.path.iter() {
            encoded_frame.extend(path_call.encode_ax25());
        }
        encoded_frame.extend(self.info.data.as_slice());
        encoded_frame
    }
}

pub struct PositionFrame {
    pub source: Callsign,
    pub destination: Callsign,
    pub path: Vec<Callsign>,
    pub info: InformationField,
    pub comment: Vec<u8>,
    pub table: Vec<u8>,
    pub symbol: u8,
    pub lat: f32,
    pub lng: f32,
    pub ambiguity: usize
}

impl PositionFrame {
    pub fn new() -> PositionFrame {
        PositionFrame {
            source: Callsign::new(),
            destination: Callsign::new(),
            path: Vec::new(),
            info: InformationField::new(),
            comment: Vec::new(),
            table: Vec::new(),
            symbol: 0,
            lat: 0.0,
            lng: 0.0,
            ambiguity: 0
        }
    }
}

impl PositionFrame {
    pub fn set_source(&mut self, source: &[u8]) {
        self.source = functions::parse_callsign(source);
    }
    pub fn set_destination(&mut self, dest: &[u8]) {
        self.destination = functions::parse_callsign(dest);
    }
    pub fn set_path(&mut self, path: Vec<Vec<u8>>) {
        for pth in path {
            let byts: &[u8] = pth.as_slice();
            self.path.push(functions::parse_callsign(byts));
        }
    }
    pub fn update_info(&mut self) {
        self.info = functions::parse_info_field(&self.create_info_field());
    }
    pub fn create_info_field(&self) -> Vec<u8> {
        let lat = geo_util::dec2dm_lat(self.lat);
        let lat_enc = geo_util::ambiguate(&lat, self.ambiguity);
        let lng = geo_util::dec2dm_lng(self.lng);
        let lng_enc = geo_util::ambiguate(&lng, self.ambiguity);

        let mut frame: Vec<u8> = Vec::new();
        frame.push(b'=');
        frame.extend(lat_enc.as_bytes());
        frame.extend(&self.table);
        frame.extend(lng_enc.as_bytes());
        frame.push(self.symbol);
        frame.extend(&self.comment);

        frame
    }
}

pub struct Callsign {
    pub callsign: String,
    pub ssid: u8,
    pub digi: bool,
}

impl Callsign {
    pub fn new() -> Callsign {
        Callsign {
            callsign: String::new(),
            ssid: 0,
            digi: false,
        }
    }
}

impl Callsign {
    pub fn set_callsign(&mut self, callsign: String) {
        self.callsign = callsign;
        // self.callsign = String::from_utf8(callsign.to_vec()).unwrap();
    }
    pub fn set_ssid(&mut self, ssid: u8) {
        self.ssid = ssid;
    }
    pub fn set_digi(&mut self, digi: bool) {
        self.digi = digi;
    }
    pub fn encode_ax25(&self) -> Vec<u8> {
        // TODO: find a way to return raw bytes
        let mut encoded_callsign: Vec<u8> = Vec::new();
        let mut encoded_ssid = (self.ssid << 1) | 0x60;
        let mut _callsign: String = String::new();

        if self.digi {
            encoded_ssid = encoded_ssid | 0x80;
        }
        for chr in self.callsign.chars() {
            _callsign.push(chr);
        }
        while _callsign.len() < 6 {
            _callsign.push(' ');
        }
        for byt in _callsign.bytes() {
            encoded_callsign.push(byt << 1);
        }
        encoded_callsign.push(encoded_ssid);

        encoded_callsign
    }
}

pub struct InformationField {
    pub data: Vec<u8>,
    pub data_type: String,
    pub safe: bool,
}

impl InformationField {
    pub fn new() -> InformationField {
        InformationField {
            data: Vec::new(),
            data_type: String::new(),
            safe: false
        }
    }
}

impl InformationField {
    pub fn set_data(&mut self, raw_data: &[u8]) {
        self.data = raw_data.to_vec();
    }
    pub fn set_data_type(&mut self, data_type: &str) {
        self.data_type = data_type.to_string();
    }
    pub fn set_safe(&mut self, safe: bool) {
        self.safe = safe;
    }
}

pub struct APRS;

pub struct TCP;

pub struct UDP;

pub struct HTTP;
<|endoftext|>


<commit_before>//! Provides helper functionality.

use std::env;
use std::process::{Command};
use std::path::{Path, PathBuf};

use libc::{c_int};

use super::{CXVersion};

//================================================
// Structs
//================================================

// try_opt! ______________________________________

macro_rules! try_opt {
    ($option:expr) => ({
        match $option {
            Some(some) => some,
            None => return None,
        }
    });
}

//================================================
// Structs
//================================================

/// A `clang` executable.
#[derive(Clone, Debug)]
pub struct Clang {
    /// The path to this `clang` executable.
    pub path: PathBuf,
    /// The version of this `clang` executable if it could be parsed.
    pub version: Option<CXVersion>,
    /// The directories searched by this `clang` executable for C headers.
    pub c_search_paths: Vec<PathBuf>,
    /// The directories searched by this `clang` executable for C++ headers.
    pub cpp_search_paths: Vec<PathBuf>,
}

impl Clang {
    //- Constructors -----------------------------

    fn new(path: PathBuf) -> Clang {
        let version = parse_version(&path);
        let c_search_paths = parse_search_paths(&path, "c");
        let cpp_search_paths = parse_search_paths(&path, "c++");
        Clang {
            path: path,
            version: version,
            c_search_paths: c_search_paths,
            cpp_search_paths: cpp_search_paths,
        }
    }

    /// Returns a `clang` executable if one can be found.
    ///
    /// If a path is supplied, that is the first directory searched. Otherwise, the directories in
    /// the system's `PATH` are searched.
    pub fn find(path: Option<&Path>) -> Option<Clang> {
        let clang = format!("clang{}", env::consts::EXE_SUFFIX);
        if let Some(path) = path {
            if contains(path, &clang) {
                return Some(Clang::new(path.join(&clang)));
            }
        }
        for path in env::split_paths(&env::var("PATH").unwrap()) {
            if contains(&path, &clang) {
                return Some(Clang::new(path.join(&clang)));
            }
        }
        None
    }
}

//================================================
// Functions
//================================================

/// Returns whether the supplied directory contains the supplied file.
fn contains(directory: &Path, file: &str) -> bool {
    Path::new(&directory).join(&file).exists()
}

/// Runs a `clang` executable, returning the output.
fn run_clang(path: &Path, arguments: &[&str], stdout: bool) -> String {
    Command::new(path.to_string_lossy().into_owned()).args(arguments).output().map(|o| {
        let output = if stdout {
            &o.stdout
        } else {
            &o.stderr
        };
        String::from_utf8_lossy(output).into_owned()
    }).unwrap()
}

/// Parses a version number if possible, ignoring trailing non-digit characters.
fn parse_version_number(number: &str) -> Option<c_int> {
    number.chars().take_while(|c| c.is_digit(10)).collect::<String>().parse().ok()
}

/// Parses the version from the output of a `clang` executable if possible.
fn parse_version(path: &Path) -> Option<CXVersion> {
    let output = run_clang(path, &["--version"], true);
    let start = try_opt!(output.find("version ")) + 8;
    let mut numbers = try_opt!(output[start..].split_whitespace().nth(0)).split(".");
    let major = try_opt!(numbers.next().and_then(parse_version_number));
    let minor = try_opt!(numbers.next().and_then(parse_version_number));
    let subminor = numbers.next().and_then(parse_version_number).unwrap_or(0);
    Some(CXVersion { Major: major, Minor: minor, Subminor: subminor })
}

/// Parses the search paths from the output of a `clang` executable.
fn parse_search_paths(path: &Path, language: &str) -> Vec<PathBuf> {
    let output = run_clang(path, &["-E", "-x", language, "-", "-v"], false);
    let start = output.find("#include <...> search starts here:").unwrap() + 34;
    let end = output.find("End of search list.").unwrap();
    output[start..end].split_whitespace().map(|l| Path::new(l.trim()).into()).collect()
}
<commit_msg>Fix clippy warnings<commit_after>//! Provides helper functionality.

use std::env;
use std::process::{Command};
use std::path::{Path, PathBuf};

use libc::{c_int};

use super::{CXVersion};

//================================================
// Structs
//================================================

// try_opt! ______________________________________

macro_rules! try_opt {
    ($option:expr) => ({
        match $option {
            Some(some) => some,
            None => return None,
        }
    });
}

//================================================
// Structs
//================================================

/// A `clang` executable.
#[derive(Clone, Debug)]
pub struct Clang {
    /// The path to this `clang` executable.
    pub path: PathBuf,
    /// The version of this `clang` executable if it could be parsed.
    pub version: Option<CXVersion>,
    /// The directories searched by this `clang` executable for C headers.
    pub c_search_paths: Vec<PathBuf>,
    /// The directories searched by this `clang` executable for C++ headers.
    pub cpp_search_paths: Vec<PathBuf>,
}

impl Clang {
    //- Constructors -----------------------------

    fn new(path: PathBuf) -> Clang {
        let version = parse_version(&path);
        let c_search_paths = parse_search_paths(&path, "c");
        let cpp_search_paths = parse_search_paths(&path, "c++");
        Clang {
            path: path,
            version: version,
            c_search_paths: c_search_paths,
            cpp_search_paths: cpp_search_paths,
        }
    }

    /// Returns a `clang` executable if one can be found.
    ///
    /// If a path is supplied, that is the first directory searched. Otherwise, the directories in
    /// the system's `PATH` are searched.
    pub fn find(path: Option<&Path>) -> Option<Clang> {
        let clang = format!("clang{}", env::consts::EXE_SUFFIX);
        if let Some(path) = path {
            if contains(path, &clang) {
                return Some(Clang::new(path.join(&clang)));
            }
        }
        for path in env::split_paths(&env::var("PATH").unwrap()) {
            if contains(&path, &clang) {
                return Some(Clang::new(path.join(&clang)));
            }
        }
        None
    }
}

//================================================
// Functions
//================================================

/// Returns whether the supplied directory contains the supplied file.
fn contains(directory: &Path, file: &str) -> bool {
    Path::new(&directory).join(&file).exists()
}

/// Runs a `clang` executable, returning the output.
fn run_clang(path: &Path, arguments: &[&str], stdout: bool) -> String {
    Command::new(path.to_string_lossy().into_owned()).args(arguments).output().map(|o| {
        let output = if stdout {
            &o.stdout
        } else {
            &o.stderr
        };
        String::from_utf8_lossy(output).into_owned()
    }).unwrap()
}

/// Parses a version number if possible, ignoring trailing non-digit characters.
fn parse_version_number(number: &str) -> Option<c_int> {
    number.chars().take_while(|c| c.is_digit(10)).collect::<String>().parse().ok()
}

/// Parses the version from the output of a `clang` executable if possible.
fn parse_version(path: &Path) -> Option<CXVersion> {
    let output = run_clang(path, &["--version"], true);
    let start = try_opt!(output.find("version ")) + 8;
    let mut numbers = try_opt!(output[start..].split_whitespace().nth(0)).split('.');
    let major = try_opt!(numbers.next().and_then(parse_version_number));
    let minor = try_opt!(numbers.next().and_then(parse_version_number));
    let subminor = numbers.next().and_then(parse_version_number).unwrap_or(0);
    Some(CXVersion { Major: major, Minor: minor, Subminor: subminor })
}

/// Parses the search paths from the output of a `clang` executable.
fn parse_search_paths(path: &Path, language: &str) -> Vec<PathBuf> {
    let output = run_clang(path, &["-E", "-x", language, "-", "-v"], false);
    let start = output.find("#include <...> search starts here:").unwrap() + 34;
    let end = output.find("End of search list.").unwrap();
    output[start..end].split_whitespace().map(|l| Path::new(l.trim()).into()).collect()
}
<|endoftext|>


<commit_before><commit_msg>Simplify two functions that deal with attributes<commit_after><|endoftext|>


<commit_before>#![crate_name = "uu_tee"]

/*
 * This file is part of the uutils coreutils package.
 *
 * (c) Aleksander Bielawski <pabzdzdzwiagief@gmail.com>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

extern crate getopts;

extern crate uucore;

use std::fs::OpenOptions;
use std::io::{copy, sink, stdin, stdout, Error, ErrorKind, Read, Result, Write};
use std::path::{Path, PathBuf};

static NAME: &str = "tee";
static VERSION: &str = env!("CARGO_PKG_VERSION");

pub fn uumain(args: Vec<String>) -> i32 {
    match options(&args).and_then(exec) {
        Ok(_) => 0,
        Err(_) => 1,
    }
}

#[allow(dead_code)]
struct Options {
    program: String,
    append: bool,
    ignore_interrupts: bool,
    print_and_exit: Option<String>,
    files: Vec<String>,
}

fn options(args: &[String]) -> Result<Options> {
    let mut opts = getopts::Options::new();

    opts.optflag("a", "append", "append to the given FILEs, do not overwrite");
    opts.optflag("i", "ignore-interrupts", "ignore interrupt signals");
    opts.optflag("h", "help", "display this help and exit");
    opts.optflag("V", "version", "output version information and exit");

    opts.parse(&args[1..])
        .map_err(|e| Error::new(ErrorKind::Other, format!("{}", e)))
        .and_then(|m| {
            let version = format!("{} {}", NAME, VERSION);
            let arguments = "[OPTION]... [FILE]...";
            let brief = "Copy standard input to each FILE, and also to standard output.";
            let comment = "If a FILE is -, it refers to a file named - .";
            let help = format!(
                "{}\n\nUsage:\n  {} {}\n\n{}\n{}",
                version,
                NAME,
                arguments,
                opts.usage(brief),
                comment
            );
            let names: Vec<String> = m.free.clone().into_iter().collect();
            let to_print = if m.opt_present("help") {
                Some(help)
            } else if m.opt_present("version") {
                Some(version)
            } else {
                None
            };
            Ok(Options {
                program: NAME.to_owned(),
                append: m.opt_present("append"),
                ignore_interrupts: m.opt_present("ignore-interrupts"),
                print_and_exit: to_print,
                files: names,
            })
        })
        .map_err(|message| warn(format!("{}", message).as_ref()))
}

fn exec(options: Options) -> Result<()> {
    match options.print_and_exit {
        Some(text) => Ok(println!("{}", text)),
        None => tee(options),
    }
}

fn tee(options: Options) -> Result<()> {
    let mut writers: Vec<Box<Write>> = options
        .files
        .clone()
        .into_iter()
        .map(|file| open(file, options.append))
        .collect();
    writers.push(Box::new(stdout()));
    let output = &mut MultiWriter { writers: writers };
    let input = &mut NamedReader {
        inner: Box::new(stdin()) as Box<Read>,
    };
    if copy(input, output).is_err() || output.flush().is_err() {
        Err(Error::new(ErrorKind::Other, ""))
    } else {
        Ok(())
    }
}

fn open(name: String, append: bool) -> Box<Write> {
    let path = PathBuf::from(name);
    let inner: Box<Write> = {
        let mut options = OpenOptions::new();
        let mode = if append {
            options.append(true)
        } else {
            options.truncate(true)
        };
        match mode.write(true).create(true).open(path.as_path()) {
            Ok(file) => Box::new(file),
            Err(_) => Box::new(sink()),
        }
    };
    Box::new(NamedWriter {
        inner: inner,
        path: path,
    }) as Box<Write>
}

struct MultiWriter {
    writers: Vec<Box<Write>>,
}

impl Write for MultiWriter {
    fn write(&mut self, buf: &[u8]) -> Result<usize> {
        for writer in &mut self.writers {
            try!(writer.write_all(buf));
        }
        Ok(buf.len())
    }

    fn flush(&mut self) -> Result<()> {
        for writer in &mut self.writers {
            try!(writer.flush());
        }
        Ok(())
    }
}

struct NamedWriter {
    inner: Box<Write>,
    path: PathBuf,
}

impl Write for NamedWriter {
    fn write(&mut self, buf: &[u8]) -> Result<usize> {
        match self.inner.write(buf) {
            Err(f) => {
                self.inner = Box::new(sink()) as Box<Write>;
                warn(format!("{}: {}", self.path.display(), f.to_string()).as_ref());
                Err(f)
            }
            okay => okay,
        }
    }

    fn flush(&mut self) -> Result<()> {
        match self.inner.flush() {
            Err(f) => {
                self.inner = Box::new(sink()) as Box<Write>;
                warn(format!("{}: {}", self.path.display(), f.to_string()).as_ref());
                Err(f)
            }
            okay => okay,
        }
    }
}

struct NamedReader {
    inner: Box<Read>,
}

impl Read for NamedReader {
    fn read(&mut self, buf: &mut [u8]) -> Result<usize> {
        match self.inner.read(buf) {
            Err(f) => {
                warn(format!("{}: {}", Path::new("stdin").display(), f.to_string()).as_ref());
                Err(f)
            }
            okay => okay,
        }
    }
}

fn warn(message: &str) -> Error {
    eprintln!("{}: {}", NAME, message);
    Error::new(ErrorKind::Other, format!("{}: {}", NAME, message))
}
<commit_msg>refactor: use shorthand fields<commit_after>#![crate_name = "uu_tee"]

/*
 * This file is part of the uutils coreutils package.
 *
 * (c) Aleksander Bielawski <pabzdzdzwiagief@gmail.com>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

extern crate getopts;

extern crate uucore;

use std::fs::OpenOptions;
use std::io::{copy, sink, stdin, stdout, Error, ErrorKind, Read, Result, Write};
use std::path::{Path, PathBuf};

static NAME: &str = "tee";
static VERSION: &str = env!("CARGO_PKG_VERSION");

pub fn uumain(args: Vec<String>) -> i32 {
    match options(&args).and_then(exec) {
        Ok(_) => 0,
        Err(_) => 1,
    }
}

#[allow(dead_code)]
struct Options {
    program: String,
    append: bool,
    ignore_interrupts: bool,
    print_and_exit: Option<String>,
    files: Vec<String>,
}

fn options(args: &[String]) -> Result<Options> {
    let mut opts = getopts::Options::new();

    opts.optflag("a", "append", "append to the given FILEs, do not overwrite");
    opts.optflag("i", "ignore-interrupts", "ignore interrupt signals");
    opts.optflag("h", "help", "display this help and exit");
    opts.optflag("V", "version", "output version information and exit");

    opts.parse(&args[1..])
        .map_err(|e| Error::new(ErrorKind::Other, format!("{}", e)))
        .and_then(|m| {
            let version = format!("{} {}", NAME, VERSION);
            let arguments = "[OPTION]... [FILE]...";
            let brief = "Copy standard input to each FILE, and also to standard output.";
            let comment = "If a FILE is -, it refers to a file named - .";
            let help = format!(
                "{}\n\nUsage:\n  {} {}\n\n{}\n{}",
                version,
                NAME,
                arguments,
                opts.usage(brief),
                comment
            );
            let names: Vec<String> = m.free.clone().into_iter().collect();
            let to_print = if m.opt_present("help") {
                Some(help)
            } else if m.opt_present("version") {
                Some(version)
            } else {
                None
            };
            Ok(Options {
                program: NAME.to_owned(),
                append: m.opt_present("append"),
                ignore_interrupts: m.opt_present("ignore-interrupts"),
                print_and_exit: to_print,
                files: names,
            })
        })
        .map_err(|message| warn(format!("{}", message).as_ref()))
}

fn exec(options: Options) -> Result<()> {
    match options.print_and_exit {
        Some(text) => Ok(println!("{}", text)),
        None => tee(options),
    }
}

fn tee(options: Options) -> Result<()> {
    let mut writers: Vec<Box<Write>> = options
        .files
        .clone()
        .into_iter()
        .map(|file| open(file, options.append))
        .collect();
    writers.push(Box::new(stdout()));
    let output = &mut MultiWriter { writers };
    let input = &mut NamedReader {
        inner: Box::new(stdin()) as Box<Read>,
    };
    if copy(input, output).is_err() || output.flush().is_err() {
        Err(Error::new(ErrorKind::Other, ""))
    } else {
        Ok(())
    }
}

fn open(name: String, append: bool) -> Box<Write> {
    let path = PathBuf::from(name);
    let inner: Box<Write> = {
        let mut options = OpenOptions::new();
        let mode = if append {
            options.append(true)
        } else {
            options.truncate(true)
        };
        match mode.write(true).create(true).open(path.as_path()) {
            Ok(file) => Box::new(file),
            Err(_) => Box::new(sink()),
        }
    };
    Box::new(NamedWriter {
        inner: inner,
        path: path,
    }) as Box<Write>
}

struct MultiWriter {
    writers: Vec<Box<Write>>,
}

impl Write for MultiWriter {
    fn write(&mut self, buf: &[u8]) -> Result<usize> {
        for writer in &mut self.writers {
            try!(writer.write_all(buf));
        }
        Ok(buf.len())
    }

    fn flush(&mut self) -> Result<()> {
        for writer in &mut self.writers {
            try!(writer.flush());
        }
        Ok(())
    }
}

struct NamedWriter {
    inner: Box<Write>,
    path: PathBuf,
}

impl Write for NamedWriter {
    fn write(&mut self, buf: &[u8]) -> Result<usize> {
        match self.inner.write(buf) {
            Err(f) => {
                self.inner = Box::new(sink()) as Box<Write>;
                warn(format!("{}: {}", self.path.display(), f.to_string()).as_ref());
                Err(f)
            }
            okay => okay,
        }
    }

    fn flush(&mut self) -> Result<()> {
        match self.inner.flush() {
            Err(f) => {
                self.inner = Box::new(sink()) as Box<Write>;
                warn(format!("{}: {}", self.path.display(), f.to_string()).as_ref());
                Err(f)
            }
            okay => okay,
        }
    }
}

struct NamedReader {
    inner: Box<Read>,
}

impl Read for NamedReader {
    fn read(&mut self, buf: &mut [u8]) -> Result<usize> {
        match self.inner.read(buf) {
            Err(f) => {
                warn(format!("{}: {}", Path::new("stdin").display(), f.to_string()).as_ref());
                Err(f)
            }
            okay => okay,
        }
    }
}

fn warn(message: &str) -> Error {
    eprintln!("{}: {}", NAME, message);
    Error::new(ErrorKind::Other, format!("{}: {}", NAME, message))
}
<|endoftext|>


<commit_before>/// 17.231s on A4-5000 (1.5 GHz)
extern crate crypto;
use crypto::digest::Digest;
use crypto::md5::Md5;

fn main() { println!("{}", get_answer("iwrupvqb")); }

/// Computes the md5_hash of a string.
fn md5_hash(input: &str) -> String {
    let mut sh = Md5::new();
    sh.input_str(input);
    String::from(sh.result_str())
}

/// Returns true if the hashed key starts with six zeroes.
fn check_hash(key: &str) -> bool { !md5_hash(key).starts_with("000000") }

/// Continually hashes keys until the first key succeeds the `check_hash()` function.
fn get_answer(input: &str) -> usize {
    (1..).skip_while(|x| check_hash(&format!("{}{:05}", input, x))).take(1).next().unwrap()
}
<commit_msg>Simplifyy Day 04<commit_after>/// 12.230s on A4-5000 (1.5 GHz)
extern crate crypto;
use crypto::digest::Digest;
use crypto::md5::Md5;

fn main() { println!("{}", get_answer("iwrupvqb")); }

/// Computes the md5_hash of a string and returns true if it starts with six zeroes.
fn hash_is_match(input: &str) -> bool {
    let mut sh = Md5::new();
    sh.input_str(input);
    sh.result_str().starts_with("000000")
}

/// Continually hashes keys until the first key succeeds the `check_hash()` function.
fn get_answer(input: &str) -> usize {
    (1..).skip_while(|x| !hash_is_match(&format!("{}{:05}", input, x))).take(1).next().unwrap()
}
<|endoftext|>


<commit_before>#![feature(box_syntax)]
#![feature(cfg_target_feature)]
//! Basic digital signal processing (DSP) operations
//!
//! Digital signal processing based on real or complex vectors in time or frequency domain. Vectors are expected to typically have a size which is at least in the order
//! of magnitude of a couple of thousand elements. This crate tries to balance between a clear API and performance in terms of processing speed.
//! This project started as small pet project to learn more about DSP, CPU architecture and Rust. Since learning
//! involves making mistakes, don't expect things to be flawless or even close to flawless.
//!
//! This library isn't suited - from my point of view - for game programming. If you are looking for vector types to do
//! 2D or 3D graphics calculations then you unfortunately have to continue with your search. However there seem to be
//! a lot of suitable crates on `crates.io` for you.
//!
//! The vector types don't distinguish between 1xN or Nx1. This is a difference to other conventions such as in MATLAB or GNU Octave.
//! The reason for this decision is that it seems to be more practical to ignore the shape of the vector.
//!
//! Right now the library uses pretty aggressive parallelization. So this means that it will keep all CPU cores busy
//! even if the performance gain is minimal e.g. because the multi core overhead is nearly as large as the performance boost
//! of multiple cores. In future there will be likely an option which tells the library how it should balance between processing time
//! and CPU utilization. The library also avoids to allocate and free memory so it allocates all of the required temporary memory when a new vector
//! is constructed. Therefore the library is likely not suitable for devices which are tight on memory. On normal desktop computers there is usually plenty of
//! memory available so that the optimization focus is on decreasing the processing time for every (common) operation and to spent little time with memory allocations.

/// Like `try!` but for operations returning a vector.
///
/// Operations which return a vector on success even return an error reason
/// together with a vector on failure. So even if the operation has failed the
/// vector can still be reused and thus memory allocation can be avoided. If
/// this is undesired then this macro can be used instead of `try!` to just
/// return the error reason.
#[macro_export]
macro_rules! try_vec {
    ( $ expr : expr ) => {
        match $expr {
            Ok(vec) => vec,
            Err((reason, _)) => return Err(reason)
        };
    };
}

extern crate simd;
extern crate num_cpus;
extern crate crossbeam;
extern crate num;
extern crate rustfft;
mod vector_types;
#[allow(dead_code)] // TODO: Remove this as soon as vector_types2 is able to replace vector_types (also rename at the same point)
mod vector_types2;
mod multicore_support;
mod simd_extensions;
pub mod window_functions;
pub mod conv_types;
pub mod combined_ops;
pub use vector_types::
    {
        DataVecDomain,
        DataVec,
        TransRes,
        VoidResult,
        ErrorReason,
        GenericVectorOps,
        RealVectorOps,
        ComplexVectorOps,
        TimeDomainOperations,
        FrequencyDomainOperations,
        SymmetricFrequencyDomainOperations,
        SymmetricTimeDomainOperations,
        GenericDataVec,
        ComplexFreqVector,
        ComplexTimeVector,
        RealTimeVector,
        RealFreqVector,
        DataVec32,
        RealTimeVector32,
        ComplexTimeVector32,
        RealFreqVector32,
        ComplexFreqVector32,
        DataVec64,
        RealTimeVector64,
        ComplexTimeVector64,
        RealFreqVector64,
        ComplexFreqVector64,
        Statistics,
        RededicateOps,
        ScaleOps,
        OffsetOps,
        DotProductOps,
        StatisticsOps,
        Convolution,
        ConvolutionOps,
        FrequencyMultiplication,
        CrossCorrelationOps,
        InterpolationOps,
        RealInterpolationOps,
        PaddingOption,
        VectorIter,
        ComplexIndex,
        ComplexIndexMut,
        RealIndex,
        RealIndexMut,
        InterleavedIndex,
        InterleavedIndexMut
    };
 pub use multicore_support::MultiCoreSettings;
 use num::traits::Float;

 /// A real floating pointer number intended to abstract over `f32` and `f64`.
 pub trait RealNumber : Float + Copy + Clone + Send + Sync { }
 impl<T> RealNumber for T
  where T: Float + Copy + Clone + Send + Sync {}

#[cfg(test)]
mod tests {
    use super::*;

    fn void_try_method() -> Result<i32, ErrorReason> {
        let array = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
        let vector = DataVec32::from_array(false, DataVecDomain::Time, &array);
        let mut dest = DataVec32::from_array(false, DataVecDomain::Time, &array);
        let _ = try!(vector.get_magnitude(&mut dest));
        Ok(0)
    }

    fn scalar_try_method() -> Result<i32, ErrorReason> {
        let array = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
        let vector = DataVec32::from_array(false, DataVecDomain::Time, &array);
        let dest = DataVec32::from_array(false, DataVecDomain::Time, &array);
        let _ = try!(vector.complex_dot_product(&dest));
        Ok(0)
    }

    fn vec_try_method() -> Result<i32, ErrorReason> {
        let array = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
        let vector = DataVec32::from_array(false, DataVecDomain::Time, &array);
        let array = [1.0, 2.0, 3.0, 4.0];
        let dest = DataVec32::from_array(false, DataVecDomain::Time, &array);
        let _ = try_vec!(vector.add(&dest));
        Ok(0)
    }

    /// This test should make sure that there are convenient error handling
    /// methods available.
    #[test]
    fn construct_real_time_vector_32_test()
    {
        let res = void_try_method();
        assert!(res.is_err());
        let res = scalar_try_method();
        assert!(res.is_err());
        let res = vec_try_method();
        assert!(res.is_err());
    }
}
<commit_msg>Added a compile time link between floating points and their SIMD register<commit_after>#![feature(box_syntax)]
#![feature(cfg_target_feature)]
//! Basic digital signal processing (DSP) operations
//!
//! Digital signal processing based on real or complex vectors in time or frequency domain. Vectors are expected to typically have a size which is at least in the order
//! of magnitude of a couple of thousand elements. This crate tries to balance between a clear API and performance in terms of processing speed.
//! This project started as small pet project to learn more about DSP, CPU architecture and Rust. Since learning
//! involves making mistakes, don't expect things to be flawless or even close to flawless.
//!
//! This library isn't suited - from my point of view - for game programming. If you are looking for vector types to do
//! 2D or 3D graphics calculations then you unfortunately have to continue with your search. However there seem to be
//! a lot of suitable crates on `crates.io` for you.
//!
//! The vector types don't distinguish between 1xN or Nx1. This is a difference to other conventions such as in MATLAB or GNU Octave.
//! The reason for this decision is that it seems to be more practical to ignore the shape of the vector.
//!
//! Right now the library uses pretty aggressive parallelization. So this means that it will keep all CPU cores busy
//! even if the performance gain is minimal e.g. because the multi core overhead is nearly as large as the performance boost
//! of multiple cores. In future there will be likely an option which tells the library how it should balance between processing time
//! and CPU utilization. The library also avoids to allocate and free memory so it allocates all of the required temporary memory when a new vector
//! is constructed. Therefore the library is likely not suitable for devices which are tight on memory. On normal desktop computers there is usually plenty of
//! memory available so that the optimization focus is on decreasing the processing time for every (common) operation and to spent little time with memory allocations.

/// Like `try!` but for operations returning a vector.
///
/// Operations which return a vector on success even return an error reason
/// together with a vector on failure. So even if the operation has failed the
/// vector can still be reused and thus memory allocation can be avoided. If
/// this is undesired then this macro can be used instead of `try!` to just
/// return the error reason.
#[macro_export]
macro_rules! try_vec {
    ( $ expr : expr ) => {
        match $expr {
            Ok(vec) => vec,
            Err((reason, _)) => return Err(reason)
        };
    };
}

extern crate simd;
extern crate num_cpus;
extern crate crossbeam;
extern crate num;
extern crate rustfft;
mod vector_types;
#[allow(dead_code)] // TODO: Remove this as soon as vector_types2 is able to replace vector_types (also rename at the same point)
mod vector_types2;
mod multicore_support;
mod simd_extensions;
pub mod window_functions;
pub mod conv_types;
pub mod combined_ops;
pub use vector_types::
    {
        DataVecDomain,
        DataVec,
        TransRes,
        VoidResult,
        ErrorReason,
        GenericVectorOps,
        RealVectorOps,
        ComplexVectorOps,
        TimeDomainOperations,
        FrequencyDomainOperations,
        SymmetricFrequencyDomainOperations,
        SymmetricTimeDomainOperations,
        GenericDataVec,
        ComplexFreqVector,
        ComplexTimeVector,
        RealTimeVector,
        RealFreqVector,
        DataVec32,
        RealTimeVector32,
        ComplexTimeVector32,
        RealFreqVector32,
        ComplexFreqVector32,
        DataVec64,
        RealTimeVector64,
        ComplexTimeVector64,
        RealFreqVector64,
        ComplexFreqVector64,
        Statistics,
        RededicateOps,
        ScaleOps,
        OffsetOps,
        DotProductOps,
        StatisticsOps,
        Convolution,
        ConvolutionOps,
        FrequencyMultiplication,
        CrossCorrelationOps,
        InterpolationOps,
        RealInterpolationOps,
        PaddingOption,
        VectorIter,
        ComplexIndex,
        ComplexIndexMut,
        RealIndex,
        RealIndexMut,
        InterleavedIndex,
        InterleavedIndexMut
    };
 pub use multicore_support::MultiCoreSettings;
 use num::traits::Float;

 use simd_extensions::*;

/// Associates a number type with a SIMD register type.
pub trait ToSimd {
    type Reg;
}

impl ToSimd for f32 {
    type Reg = Reg32;
}

impl ToSimd for f64 {
    type Reg = Reg64;
}

 /// A real floating pointer number intended to abstract over `f32` and `f64`.
 pub trait RealNumber : Float + Copy + Clone + Send + Sync + ToSimd { }
 impl<T> RealNumber for T
  where T: Float + Copy + Clone + Send + Sync + ToSimd,
           <T as ToSimd>::Reg: Simd<T> + SimdGeneric<T> {}

#[cfg(test)]
mod tests {
    use super::*;
    use simd_extensions::Simd;

    fn void_try_method() -> Result<i32, ErrorReason> {
        let array = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
        let vector = DataVec32::from_array(false, DataVecDomain::Time, &array);
        let mut dest = DataVec32::from_array(false, DataVecDomain::Time, &array);
        let _ = try!(vector.get_magnitude(&mut dest));
        Ok(0)
    }

    fn scalar_try_method() -> Result<i32, ErrorReason> {
        let array = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
        let vector = DataVec32::from_array(false, DataVecDomain::Time, &array);
        let dest = DataVec32::from_array(false, DataVecDomain::Time, &array);
        let _ = try!(vector.complex_dot_product(&dest));
        Ok(0)
    }

    fn vec_try_method() -> Result<i32, ErrorReason> {
        let array = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
        let vector = DataVec32::from_array(false, DataVecDomain::Time, &array);
        let array = [1.0, 2.0, 3.0, 4.0];
        let dest = DataVec32::from_array(false, DataVecDomain::Time, &array);
        let _ = try_vec!(vector.add(&dest));
        Ok(0)
    }

    /// This test should make sure that there are convenient error handling
    /// methods available.
    #[test]
    fn construct_real_time_vector_32_test()
    {
        let res = void_try_method();
        assert!(res.is_err());
        let res = scalar_try_method();
        assert!(res.is_err());
        let res = vec_try_method();
        assert!(res.is_err());
    }

    #[test]
    fn to_simd_test()
    {
        // This is more a check for syntax. So if it compiles
        // then the test already passes. The assert is then only
        // a sanity check.
        let reg = <f32 as ToSimd>::Reg::splat(1.0);
        let sum = reg.sum_real();
        assert!(sum > 0.0);
    }
}
<|endoftext|>


<commit_before>#![no_main]

use arbitrary::Arbitrary;
use libfuzzer_sys::fuzz_target;

#[derive(Arbitrary, Debug)]
enum IntegerInput {
    I8(i8),
    U8(u8),
    I16(i16),
    U16(u16),
    I32(i32),
    U32(u32),
    I64(i64),
    U64(u64),
    ISIZE(isize),
    USIZE(usize),
    I128(i128),
    U128(u128),
}

fuzz_target!(|input: IntegerInput| {
    let mut buffer = itoa::Buffer::new();
    match input {
        IntegerInput::I8(val) => buffer.format(val),
        IntegerInput::U8(val) => buffer.format(val),
        IntegerInput::I16(val) => buffer.format(val),
        IntegerInput::U16(val) => buffer.format(val),
        IntegerInput::I32(val) => buffer.format(val),
        IntegerInput::U32(val) => buffer.format(val),
        IntegerInput::I64(val) => buffer.format(val),
        IntegerInput::U64(val) => buffer.format(val),
        IntegerInput::ISIZE(val) => buffer.format(val),
        IntegerInput::USIZE(val) => buffer.format(val),
        IntegerInput::I128(val) => buffer.format(val),
        IntegerInput::U128(val) => buffer.format(val),
    };
});
<commit_msg>Move ptr sized integer variant last<commit_after>#![no_main]

use arbitrary::Arbitrary;
use libfuzzer_sys::fuzz_target;

#[derive(Arbitrary, Debug)]
enum IntegerInput {
    I8(i8),
    U8(u8),
    I16(i16),
    U16(u16),
    I32(i32),
    U32(u32),
    I64(i64),
    U64(u64),
    I128(i128),
    U128(u128),
    ISIZE(isize),
    USIZE(usize),
}

fuzz_target!(|input: IntegerInput| {
    let mut buffer = itoa::Buffer::new();
    match input {
        IntegerInput::I8(val) => buffer.format(val),
        IntegerInput::U8(val) => buffer.format(val),
        IntegerInput::I16(val) => buffer.format(val),
        IntegerInput::U16(val) => buffer.format(val),
        IntegerInput::I32(val) => buffer.format(val),
        IntegerInput::U32(val) => buffer.format(val),
        IntegerInput::I64(val) => buffer.format(val),
        IntegerInput::U64(val) => buffer.format(val),
        IntegerInput::I128(val) => buffer.format(val),
        IntegerInput::U128(val) => buffer.format(val),
        IntegerInput::ISIZE(val) => buffer.format(val),
        IntegerInput::USIZE(val) => buffer.format(val),
    };
});
<|endoftext|>


<commit_before>use std::collections::HashMap;

use rand;
use rand::Rng;
use rand::distributions::IndependentSample;
use itertools::Itertools;
use ndarray::prelude::*;
use ordered_float::NotNaN;

use bio::stats::{Prob, LogProb};

use io::codebook::{Codebook, FeatureID};

use model::readout::{Expressions, Counts, Miscalls, FeatureModel, NoiseModel, Xi};
use model::readout::feature_model::AbstractFeatureModel;

pub struct JointModel {
    feature_models: HashMap<FeatureID, FeatureModel>,
    noise_model: NoiseModel,
    expressions: Expressions,
    miscalls_exact: Miscalls,
    miscalls_mismatch: Miscalls,
    margin: u32,
    em_run: bool,
    rng: rand::StdRng
}

impl JointModel {
    pub fn new<'a, I: Iterator<Item=(&'a String, &'a Counts)>>(
        counts: I,
        p0: &[Prob],
        p1: &[Prob],
        codebook: &Codebook,
        window_width: u32
    ) -> Self {
        let xi = Xi::new(p0, p1);
        let mut rng = rand::StdRng::new().unwrap();

        // Generate feature models and info about not expressed features.
        let mut feature_models = HashMap::new();
        let mut not_expressed_counts = Vec::new();
        let mut not_expressed_feature_ids = Vec::new();
        for (feature, counts) in counts {
            let feature = codebook.get_id(feature);
            let rec = codebook.record(feature);
            if rec.expressed() {
                feature_models.insert(
                    feature, FeatureModel::new(feature, counts.clone(), codebook, &xi)
                );
            } else {
                not_expressed_counts.push(counts.clone());
                not_expressed_feature_ids.push(feature);
            }
        }

        let feature_count = codebook.len() + 1;

        // calculate start values (we take random numbers as start expression)
        let start_range = rand::distributions::Range::new(1, 10000);
        let mut expressions = Array1::from_iter((0..feature_count).map(
            |_| start_range.ind_sample(&mut rng)
        ));
        for &feat_id in &not_expressed_feature_ids {
            expressions[feat_id] = 0;
        }

        // Take value larger than maximum feature id as noise id.
        let noise_id = codebook.len();
        let noise_model = NoiseModel::new(
            noise_id, not_expressed_feature_ids, not_expressed_counts, codebook, &xi
        );

        let miscalls_exact = Miscalls::new(feature_count, &feature_models, &noise_model, true);
        let miscalls_mismatch = Miscalls::new(feature_count, &feature_models, &noise_model, false);

        JointModel {
            feature_models: feature_models,
            noise_model: noise_model,
            expressions: expressions,
            miscalls_exact: miscalls_exact,
            miscalls_mismatch: miscalls_mismatch,
            margin: window_width / 2,
            em_run: false,
            rng: rng
        }
    }

    /// Jointly estimate MAP using the EM algorithm.
    ///
    /// Expressions are our model parameters to estimate, miscalls are our latent variables,
    /// counts are our observed variables.
    ///
    /// # Arguments
    ///
    /// * ID of cell that is investigated (for debugging purposes).
    pub fn expectation_maximization(&mut self, cell: &str) {
        let mut feature_models: Vec<Box<&AbstractFeatureModel>> = self.feature_models.values().map(
            |m| Box::new(m as &AbstractFeatureModel)
        ).collect_vec();
        //feature_models.push(Box::new(&self.noise_model));

        let n_iterations = 100;
        let change_window = 10;

        let mut last_changes = Array2::from_elem((self.expressions.len(), change_window), 0.0);

        // EM iterations
        for i in 1..n_iterations + 1 {
            debug!("EM-iteration {} of at most {}", i, n_iterations);

            // E-step: estimate miscalls
            debug!("E-step");

            // Shuffle models, such that miscalls can be drawn unbiased
            // this is necessary because there is a maximum miscall count for each feature.
            // If we would not shuffle, it would be easier for the first model to provide miscalls.
            self.rng.shuffle(&mut feature_models);
            for m in &feature_models {
                m.mle_miscalls(
                    &self.expressions,
                    &mut self.miscalls_exact,
                    &mut self.miscalls_mismatch,
                    &mut self.rng
                );
            }

            // M-step: estimate expressions that maximize the probability
            debug!("M-step");
            for m in &feature_models {
                let feature_id = m.feature_id();
                last_changes[(feature_id, feature_id % change_window)] = m.mle_expression(
                    &mut self.expressions,
                    &self.miscalls_exact,
                    &self.miscalls_mismatch
                ) as f64;
            }

            // calculate mean change per feature of last 5 steps
            let mean_changes = last_changes.mean(Axis(1));
            debug!("TYPE=EM-iteration, CELL={}, x={:?}", cell, self.expressions);
            debug!("mean changes={:?}", mean_changes);
            let convergence = *mean_changes.iter().map(
                |&c| NotNaN::new(c.abs()).unwrap()
            ).max().unwrap() <= 1.0;

            if i >= change_window && convergence {
                debug!("Convergence reached, stopping EM algorithm.");
                break;
            }
        }
        self.em_run = true;
        //debug!("TYPE=EM-result, CELL={}, noise-rate={}", cell, self.noise_rate());
    }

    pub fn noise_rate(&self) -> f64 {
        assert!(self.em_run);
        self.expressions[self.expressions.len() - 1] as f64 /
        self.expressions.slice(s![..-1]).scalar_sum() as f64
    }

    pub fn likelihood(&mut self, feature_id: FeatureID, x: u32) -> LogProb {
        assert!(self.em_run);
        self.feature_model(feature_id).likelihood(
            x, &self.miscalls_exact, &self.miscalls_mismatch
        )
    }

    pub fn map_estimate(&self, feature_id: FeatureID) -> u32 {
        assert!(self.em_run);
        self.expressions[feature_id as usize]
    }

    fn feature_model(&self, feature_id: FeatureID) -> &FeatureModel {
        self.feature_models.get(&feature_id).unwrap()
    }

    /// Prior window for calculating expression PMF
    pub fn window(&self, feature_id: FeatureID) -> (u32, u32) {
        assert!(self.em_run);
        let x = self.map_estimate(feature_id);
        let xmin = self.feature_model(feature_id).min_expression(
            &self.miscalls_exact, &self.miscalls_mismatch
        );
        let xmax = x + self.margin;
        assert!(xmax >= xmin, "xmax has to be greater than xmin: xmin={}, xmax={}", xmin, xmax);
        (xmin, xmax)
    }
}
<commit_msg>Enable noise.<commit_after>use std::collections::HashMap;

use rand;
use rand::Rng;
use rand::distributions::IndependentSample;
use itertools::Itertools;
use ndarray::prelude::*;
use ordered_float::NotNaN;

use bio::stats::{Prob, LogProb};

use io::codebook::{Codebook, FeatureID};

use model::readout::{Expressions, Counts, Miscalls, FeatureModel, NoiseModel, Xi};
use model::readout::feature_model::AbstractFeatureModel;

pub struct JointModel {
    feature_models: HashMap<FeatureID, FeatureModel>,
    noise_model: NoiseModel,
    expressions: Expressions,
    miscalls_exact: Miscalls,
    miscalls_mismatch: Miscalls,
    margin: u32,
    em_run: bool,
    rng: rand::StdRng
}

impl JointModel {
    pub fn new<'a, I: Iterator<Item=(&'a String, &'a Counts)>>(
        counts: I,
        p0: &[Prob],
        p1: &[Prob],
        codebook: &Codebook,
        window_width: u32
    ) -> Self {
        let xi = Xi::new(p0, p1);
        let mut rng = rand::StdRng::new().unwrap();

        // Generate feature models and info about not expressed features.
        let mut feature_models = HashMap::new();
        let mut not_expressed_counts = Vec::new();
        let mut not_expressed_feature_ids = Vec::new();
        for (feature, counts) in counts {
            let feature = codebook.get_id(feature);
            let rec = codebook.record(feature);
            if rec.expressed() {
                feature_models.insert(
                    feature, FeatureModel::new(feature, counts.clone(), codebook, &xi)
                );
            } else {
                not_expressed_counts.push(counts.clone());
                not_expressed_feature_ids.push(feature);
            }
        }

        let feature_count = codebook.len() + 1;

        // calculate start values (we take random numbers as start expression)
        let start_range = rand::distributions::Range::new(1, 10000);
        let mut expressions = Array1::from_iter((0..feature_count).map(
            |_| start_range.ind_sample(&mut rng)
        ));
        for &feat_id in &not_expressed_feature_ids {
            expressions[feat_id] = 0;
        }

        // Take value larger than maximum feature id as noise id.
        let noise_id = codebook.len();
        let noise_model = NoiseModel::new(
            noise_id, not_expressed_feature_ids, not_expressed_counts, codebook, &xi
        );

        let miscalls_exact = Miscalls::new(feature_count, &feature_models, &noise_model, true);
        let miscalls_mismatch = Miscalls::new(feature_count, &feature_models, &noise_model, false);

        JointModel {
            feature_models: feature_models,
            noise_model: noise_model,
            expressions: expressions,
            miscalls_exact: miscalls_exact,
            miscalls_mismatch: miscalls_mismatch,
            margin: window_width / 2,
            em_run: false,
            rng: rng
        }
    }

    /// Jointly estimate MAP using the EM algorithm.
    ///
    /// Expressions are our model parameters to estimate, miscalls are our latent variables,
    /// counts are our observed variables.
    ///
    /// # Arguments
    ///
    /// * ID of cell that is investigated (for debugging purposes).
    pub fn expectation_maximization(&mut self, cell: &str) {
        let mut feature_models: Vec<Box<&AbstractFeatureModel>> = self.feature_models.values().map(
            |m| Box::new(m as &AbstractFeatureModel)
        ).collect_vec();
        feature_models.push(Box::new(&self.noise_model));

        let n_iterations = 100;
        let change_window = 10;

        let mut last_changes = Array2::from_elem((self.expressions.len(), change_window), 0.0);

        // EM iterations
        for i in 1..n_iterations + 1 {
            debug!("EM-iteration {} of at most {}", i, n_iterations);

            // E-step: estimate miscalls
            debug!("E-step");

            // Shuffle models, such that miscalls can be drawn unbiased
            // this is necessary because there is a maximum miscall count for each feature.
            // If we would not shuffle, it would be easier for the first model to provide miscalls.
            self.rng.shuffle(&mut feature_models);
            for m in &feature_models {
                m.mle_miscalls(
                    &self.expressions,
                    &mut self.miscalls_exact,
                    &mut self.miscalls_mismatch,
                    &mut self.rng
                );
            }

            // M-step: estimate expressions that maximize the probability
            debug!("M-step");
            for m in &feature_models {
                let feature_id = m.feature_id();
                last_changes[(feature_id, feature_id % change_window)] = m.mle_expression(
                    &mut self.expressions,
                    &self.miscalls_exact,
                    &self.miscalls_mismatch
                ) as f64;
            }

            // calculate mean change per feature of last 5 steps
            let mean_changes = last_changes.mean(Axis(1));
            debug!("TYPE=EM-iteration, CELL={}, x={:?}", cell, self.expressions);
            debug!("mean changes={:?}", mean_changes);
            let convergence = *mean_changes.iter().map(
                |&c| NotNaN::new(c.abs()).unwrap()
            ).max().unwrap() <= 1.0;

            if i >= change_window && convergence {
                debug!("Convergence reached, stopping EM algorithm.");
                break;
            }
        }
        self.em_run = true;
        debug!("TYPE=EM-result, CELL={}, noise-rate={}", cell, self.noise_rate());
    }

    pub fn noise_rate(&self) -> f64 {
        assert!(self.em_run);
        self.expressions[self.expressions.len() - 1] as f64 /
        self.expressions.slice(s![..-1]).scalar_sum() as f64
    }

    pub fn likelihood(&mut self, feature_id: FeatureID, x: u32) -> LogProb {
        assert!(self.em_run);
        self.feature_model(feature_id).likelihood(
            x, &self.miscalls_exact, &self.miscalls_mismatch
        )
    }

    pub fn map_estimate(&self, feature_id: FeatureID) -> u32 {
        assert!(self.em_run);
        self.expressions[feature_id as usize]
    }

    fn feature_model(&self, feature_id: FeatureID) -> &FeatureModel {
        self.feature_models.get(&feature_id).unwrap()
    }

    /// Prior window for calculating expression PMF
    pub fn window(&self, feature_id: FeatureID) -> (u32, u32) {
        assert!(self.em_run);
        let x = self.map_estimate(feature_id);
        let xmin = self.feature_model(feature_id).min_expression(
            &self.miscalls_exact, &self.miscalls_mismatch
        );
        let xmax = x + self.margin;
        assert!(xmax >= xmin, "xmax has to be greater than xmin: xmin={}, xmax={}", xmin, xmax);
        (xmin, xmax)
    }
}
<|endoftext|>


<commit_before>use crate::metrics;
use beacon_chain::{BeaconChain, BeaconChainTypes};
use environment::RuntimeContext;
use eth2_libp2p::NetworkGlobals;
use futures::{Future, Stream};
use parking_lot::Mutex;
use slog::{debug, error, info, warn};
use slot_clock::SlotClock;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::timer::Interval;
use types::{EthSpec, Slot};

/// Create a warning log whenever the peer count is at or below this value.
pub const WARN_PEER_COUNT: usize = 1;

const SECS_PER_MINUTE: f64 = 60.0;
const SECS_PER_HOUR: f64 = 3600.0;
const SECS_PER_DAY: f64 = 86400.0; // non-leap
const SECS_PER_WEEK: f64 = 604_800.0; // non-leap
const DAYS_PER_WEEK: f64 = 7.0;
const HOURS_PER_DAY: f64 = 24.0;
const MINUTES_PER_HOUR: f64 = 60.0;

/// The number of historical observations that should be used to determine the average sync time.
const SPEEDO_OBSERVATIONS: usize = 4;

/// Spawns a notifier service which periodically logs information about the node.
pub fn spawn_notifier<T: BeaconChainTypes>(
    context: RuntimeContext<T::EthSpec>,
    beacon_chain: Arc<BeaconChain<T>>,
    network: Arc<NetworkGlobals<T::EthSpec>>,
    milliseconds_per_slot: u64,
) -> Result<tokio::sync::oneshot::Sender<()>, String> {
    let log_1 = context.log.clone();
    let log_2 = context.log.clone();
    let log_3 = context.log.clone();

    let slot_duration = Duration::from_millis(milliseconds_per_slot);
    let duration_to_next_slot = beacon_chain
        .slot_clock
        .duration_to_next_slot()
        .ok_or_else(|| "slot_notifier unable to determine time to next slot")?;

    // Run this half way through each slot.
    let start_instant = Instant::now() + duration_to_next_slot + (slot_duration / 2);

    // Run this each slot.
    let interval_duration = slot_duration;

    let speedo = Mutex::new(Speedo::default());

    let interval_future = Interval::new(start_instant, interval_duration)
        .map_err(
            move |e| error!(log_1, "Slot notifier timer failed"; "error" => format!("{:?}", e)),
        )
        .for_each(move |_| {
            let log = log_2.clone();

            let connected_peer_count = network.connected_peers();
            let sync_state = network.sync_state();

            let head_info = beacon_chain.head_info()
                .map_err(|e| error!(
                    log,
                    "Failed to get beacon chain head info";
                    "error" => format!("{:?}", e)
                ))?;

            let head_slot = head_info.slot;
            let current_slot = beacon_chain.slot().map_err(|e| {
                error!(
                    log,
                    "Unable to read current slot";
                    "error" => format!("{:?}", e)
                )
            })?;
            let current_epoch = current_slot.epoch(T::EthSpec::slots_per_epoch());
            let finalized_epoch = head_info.finalized_checkpoint.epoch;
            let finalized_root = head_info.finalized_checkpoint.root;
            let head_root = head_info.block_root;

            let mut speedo = speedo.lock();
            speedo.observe(head_slot, Instant::now());

            metrics::set_gauge(&metrics::SYNC_SLOTS_PER_SECOND, speedo.slots_per_second().unwrap_or_else(|| 0_f64) as i64);

            // The next two lines take advantage of saturating subtraction on `Slot`.
            let head_distance = current_slot - head_slot;

            if connected_peer_count <= WARN_PEER_COUNT {
                warn!(log, "Low peer count"; "peer_count" => peer_count_pretty(connected_peer_count));
            }

            debug!(
                log,
                "Slot timer";
                "peers" => peer_count_pretty(connected_peer_count),
                "finalized_root" => format!("{}", finalized_root),
                "finalized_epoch" => finalized_epoch,
                "head_block" => format!("{}", head_root),
                "head_slot" => head_slot,
                "current_slot" => current_slot,
                "sync_state" =>format!("{}", sync_state)
            );

            // Log if we are syncing
            if sync_state.is_syncing() {
                let distance = format!(
                    "{} slots ({})",
                    head_distance.as_u64(),
                    slot_distance_pretty(head_distance, slot_duration)
                );
                info!(
                    log,
                    "Syncing";
                    "peers" => peer_count_pretty(connected_peer_count),
                    "distance" => distance,
                    "speed" => sync_speed_pretty(speedo.slots_per_second()),
                    "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
                );
            } else {
                if sync_state.is_synced() {
                    info!(
                        log_2,
                        "Synced";
                        "peers" => peer_count_pretty(connected_peer_count),
                        "finalized_root" => format!("{}", finalized_root),
                        "finalized_epoch" => finalized_epoch,
                        "epoch" => current_epoch,
                        "slot" => current_slot,
                    );
                } else {
                    info!(
                        log_2,
                        "Searching for peers";
                        "peers" => peer_count_pretty(connected_peer_count),
                        "finalized_root" => format!("{}", finalized_root),
                        "finalized_epoch" => finalized_epoch,
                        "head_slot" => head_slot,
                        "current_slot" => current_slot,
                    );
                }
            }
            Ok(())
        })
        .then(move |result| {
            match result {
                Ok(()) => Ok(()),
                Err(e) => {
                    error!(
                    log_3,
                    "Notifier failed to notify";
                    "error" => format!("{:?}", e)
                );
                Ok(())
            } } });

    let (exit_signal, exit) = tokio::sync::oneshot::channel();

    context
        .executor
        .spawn(interval_future.select(exit).map(|_| ()).map_err(|_| ()));

    Ok(exit_signal)
}

/// Returns the peer count, returning something helpful if it's `usize::max_value` (effectively a
/// `None` value).
fn peer_count_pretty(peer_count: usize) -> String {
    if peer_count == usize::max_value() {
        String::from("--")
    } else {
        format!("{}", peer_count)
    }
}

/// Returns a nicely formatted string describing the rate of slot imports per second.
fn sync_speed_pretty(slots_per_second: Option<f64>) -> String {
    if let Some(slots_per_second) = slots_per_second {
        format!("{:.2} slots/sec", slots_per_second)
    } else {
        "--".into()
    }
}

/// Returns a nicely formatted string how long will we reach the target slot.
fn estimated_time_pretty(seconds_till_slot: Option<f64>) -> String {
    if let Some(seconds_till_slot) = seconds_till_slot {
        seconds_pretty(seconds_till_slot)
    } else {
        "--".into()
    }
}

/// Returns a nicely formatted string describing the `slot_span` in terms of weeks, days, hours
/// and/or minutes.
fn slot_distance_pretty(slot_span: Slot, slot_duration: Duration) -> String {
    if slot_duration == Duration::from_secs(0) {
        return String::from("Unknown");
    }

    let secs = (slot_duration * slot_span.as_u64() as u32).as_secs();
    seconds_pretty(secs as f64)
}

/// Returns a nicely formatted string describing the `slot_span` in terms of weeks, days, hours
/// and/or minutes.
fn seconds_pretty(secs: f64) -> String {
    if secs <= 0.0 {
        return "--".into();
    }

    let weeks = secs / SECS_PER_WEEK;
    let days = secs / SECS_PER_DAY;
    let hours = secs / SECS_PER_HOUR;
    let minutes = secs / SECS_PER_MINUTE;

    if weeks.floor() > 0.0 {
        format!(
            "{:.0} weeks {:.0} days",
            weeks,
            (days % DAYS_PER_WEEK).round()
        )
    } else if days.floor() > 0.0 {
        format!(
            "{:.0} days {:.0} hrs",
            days,
            (hours % HOURS_PER_DAY).round()
        )
    } else if hours.floor() > 0.0 {
        format!(
            "{:.0} hrs {:.0} mins",
            hours,
            (minutes % MINUTES_PER_HOUR).round()
        )
    } else {
        format!("{:.0} mins", minutes.round())
    }
}

/// "Speedo" is Australian for speedometer. This struct observes syncing times.
#[derive(Default)]
pub struct Speedo(Vec<(Slot, Instant)>);

impl Speedo {
    /// Observe that we were at some `slot` at the given `instant`.
    pub fn observe(&mut self, slot: Slot, instant: Instant) {
        if self.0.len() > SPEEDO_OBSERVATIONS {
            self.0.remove(0);
        }

        self.0.push((slot, instant));
    }

    /// Returns the average of the speeds between each observation.
    ///
    /// Does not gracefully handle slots that are above `u32::max_value()`.
    pub fn slots_per_second(&self) -> Option<f64> {
        let speeds = self
            .0
            .windows(2)
            .filter_map(|windows| {
                let (slot_a, instant_a) = windows[0];
                let (slot_b, instant_b) = windows[1];

                // Taking advantage of saturating subtraction on `Slot`.
                let distance = f64::from((slot_b - slot_a).as_u64() as u32);

                let seconds = f64::from((instant_b - instant_a).as_millis() as u32) / 1_000.0;

                if seconds > 0.0 {
                    Some(distance / seconds)
                } else {
                    None
                }
            })
            .collect::<Vec<f64>>();

        let count = speeds.len();
        let sum: f64 = speeds.iter().sum();

        if count > 0 {
            Some(sum / f64::from(count as u32))
        } else {
            None
        }
    }

    /// Returns the time we should reach the given `slot`, judging by the latest observation and
    /// historical average syncing time.
    ///
    /// Returns `None` if the slot is prior to our latest observed slot or we have not made any
    /// observations.
    pub fn estimated_time_till_slot(&self, target_slot: Slot) -> Option<f64> {
        let (prev_slot, _) = self.0.last()?;
        let slots_per_second = self.slots_per_second()?;

        if target_slot > *prev_slot && slots_per_second > 0.0 {
            let distance = (target_slot - *prev_slot).as_u64() as f64;
            Some(distance / slots_per_second)
        } else {
            None
        }
    }
}
<commit_msg>beacon/notifier: display block information for current slot (#1084)<commit_after>use crate::metrics;
use beacon_chain::{BeaconChain, BeaconChainTypes};
use environment::RuntimeContext;
use eth2_libp2p::NetworkGlobals;
use futures::{Future, Stream};
use parking_lot::Mutex;
use slog::{debug, error, info, warn};
use slot_clock::SlotClock;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::timer::Interval;
use types::{EthSpec, Slot};

/// Create a warning log whenever the peer count is at or below this value.
pub const WARN_PEER_COUNT: usize = 1;

const SECS_PER_MINUTE: f64 = 60.0;
const SECS_PER_HOUR: f64 = 3600.0;
const SECS_PER_DAY: f64 = 86400.0; // non-leap
const SECS_PER_WEEK: f64 = 604_800.0; // non-leap
const DAYS_PER_WEEK: f64 = 7.0;
const HOURS_PER_DAY: f64 = 24.0;
const MINUTES_PER_HOUR: f64 = 60.0;

/// The number of historical observations that should be used to determine the average sync time.
const SPEEDO_OBSERVATIONS: usize = 4;

/// Spawns a notifier service which periodically logs information about the node.
pub fn spawn_notifier<T: BeaconChainTypes>(
    context: RuntimeContext<T::EthSpec>,
    beacon_chain: Arc<BeaconChain<T>>,
    network: Arc<NetworkGlobals<T::EthSpec>>,
    milliseconds_per_slot: u64,
) -> Result<tokio::sync::oneshot::Sender<()>, String> {
    let log_1 = context.log.clone();
    let log_2 = context.log.clone();
    let log_3 = context.log.clone();

    let slot_duration = Duration::from_millis(milliseconds_per_slot);
    let duration_to_next_slot = beacon_chain
        .slot_clock
        .duration_to_next_slot()
        .ok_or_else(|| "slot_notifier unable to determine time to next slot")?;

    // Run this half way through each slot.
    let start_instant = Instant::now() + duration_to_next_slot + (slot_duration / 2);

    // Run this each slot.
    let interval_duration = slot_duration;

    let speedo = Mutex::new(Speedo::default());

    let interval_future = Interval::new(start_instant, interval_duration)
        .map_err(
            move |e| error!(log_1, "Slot notifier timer failed"; "error" => format!("{:?}", e)),
        )
        .for_each(move |_| {
            let log = log_2.clone();

            let connected_peer_count = network.connected_peers();
            let sync_state = network.sync_state();

            let head_info = beacon_chain.head_info()
                .map_err(|e| error!(
                    log,
                    "Failed to get beacon chain head info";
                    "error" => format!("{:?}", e)
                ))?;

            let head_slot = head_info.slot;
            let current_slot = beacon_chain.slot().map_err(|e| {
                error!(
                    log,
                    "Unable to read current slot";
                    "error" => format!("{:?}", e)
                )
            })?;
            let current_epoch = current_slot.epoch(T::EthSpec::slots_per_epoch());
            let finalized_epoch = head_info.finalized_checkpoint.epoch;
            let finalized_root = head_info.finalized_checkpoint.root;
            let head_root = head_info.block_root;

            let mut speedo = speedo.lock();
            speedo.observe(head_slot, Instant::now());

            metrics::set_gauge(&metrics::SYNC_SLOTS_PER_SECOND, speedo.slots_per_second().unwrap_or_else(|| 0_f64) as i64);

            // The next two lines take advantage of saturating subtraction on `Slot`.
            let head_distance = current_slot - head_slot;

            if connected_peer_count <= WARN_PEER_COUNT {
                warn!(log, "Low peer count"; "peer_count" => peer_count_pretty(connected_peer_count));
            }

            debug!(
                log,
                "Slot timer";
                "peers" => peer_count_pretty(connected_peer_count),
                "finalized_root" => format!("{}", finalized_root),
                "finalized_epoch" => finalized_epoch,
                "head_block" => format!("{}", head_root),
                "head_slot" => head_slot,
                "current_slot" => current_slot,
                "sync_state" =>format!("{}", sync_state)
            );

            // Log if we are syncing
            if sync_state.is_syncing() {
                let distance = format!(
                    "{} slots ({})",
                    head_distance.as_u64(),
                    slot_distance_pretty(head_distance, slot_duration)
                );
                info!(
                    log,
                    "Syncing";
                    "peers" => peer_count_pretty(connected_peer_count),
                    "distance" => distance,
                    "speed" => sync_speed_pretty(speedo.slots_per_second()),
                    "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
                );
            } else {
                if sync_state.is_synced() {
                    let block_info = if current_slot > head_slot { format!("   …  empty") } else { format!("{}", head_root) };
                    info!(
                        log_2,
                        "Synced";
                        "peers" => peer_count_pretty(connected_peer_count),
                        "finalized_root" => format!("{}", finalized_root),
                        "finalized_epoch" => finalized_epoch,
                        "epoch" => current_epoch,
                        "block" => block_info,
                        "slot" => current_slot,
                    );
                } else {
                    info!(
                        log_2,
                        "Searching for peers";
                        "peers" => peer_count_pretty(connected_peer_count),
                        "finalized_root" => format!("{}", finalized_root),
                        "finalized_epoch" => finalized_epoch,
                        "head_slot" => head_slot,
                        "current_slot" => current_slot,
                    );
                }
            }
            Ok(())
        })
        .then(move |result| {
            match result {
                Ok(()) => Ok(()),
                Err(e) => {
                    error!(
                    log_3,
                    "Notifier failed to notify";
                    "error" => format!("{:?}", e)
                );
                Ok(())
            } } });

    let (exit_signal, exit) = tokio::sync::oneshot::channel();

    context
        .executor
        .spawn(interval_future.select(exit).map(|_| ()).map_err(|_| ()));

    Ok(exit_signal)
}

/// Returns the peer count, returning something helpful if it's `usize::max_value` (effectively a
/// `None` value).
fn peer_count_pretty(peer_count: usize) -> String {
    if peer_count == usize::max_value() {
        String::from("--")
    } else {
        format!("{}", peer_count)
    }
}

/// Returns a nicely formatted string describing the rate of slot imports per second.
fn sync_speed_pretty(slots_per_second: Option<f64>) -> String {
    if let Some(slots_per_second) = slots_per_second {
        format!("{:.2} slots/sec", slots_per_second)
    } else {
        "--".into()
    }
}

/// Returns a nicely formatted string how long will we reach the target slot.
fn estimated_time_pretty(seconds_till_slot: Option<f64>) -> String {
    if let Some(seconds_till_slot) = seconds_till_slot {
        seconds_pretty(seconds_till_slot)
    } else {
        "--".into()
    }
}

/// Returns a nicely formatted string describing the `slot_span` in terms of weeks, days, hours
/// and/or minutes.
fn slot_distance_pretty(slot_span: Slot, slot_duration: Duration) -> String {
    if slot_duration == Duration::from_secs(0) {
        return String::from("Unknown");
    }

    let secs = (slot_duration * slot_span.as_u64() as u32).as_secs();
    seconds_pretty(secs as f64)
}

/// Returns a nicely formatted string describing the `slot_span` in terms of weeks, days, hours
/// and/or minutes.
fn seconds_pretty(secs: f64) -> String {
    if secs <= 0.0 {
        return "--".into();
    }

    let weeks = secs / SECS_PER_WEEK;
    let days = secs / SECS_PER_DAY;
    let hours = secs / SECS_PER_HOUR;
    let minutes = secs / SECS_PER_MINUTE;

    if weeks.floor() > 0.0 {
        format!(
            "{:.0} weeks {:.0} days",
            weeks,
            (days % DAYS_PER_WEEK).round()
        )
    } else if days.floor() > 0.0 {
        format!(
            "{:.0} days {:.0} hrs",
            days,
            (hours % HOURS_PER_DAY).round()
        )
    } else if hours.floor() > 0.0 {
        format!(
            "{:.0} hrs {:.0} mins",
            hours,
            (minutes % MINUTES_PER_HOUR).round()
        )
    } else {
        format!("{:.0} mins", minutes.round())
    }
}

/// "Speedo" is Australian for speedometer. This struct observes syncing times.
#[derive(Default)]
pub struct Speedo(Vec<(Slot, Instant)>);

impl Speedo {
    /// Observe that we were at some `slot` at the given `instant`.
    pub fn observe(&mut self, slot: Slot, instant: Instant) {
        if self.0.len() > SPEEDO_OBSERVATIONS {
            self.0.remove(0);
        }

        self.0.push((slot, instant));
    }

    /// Returns the average of the speeds between each observation.
    ///
    /// Does not gracefully handle slots that are above `u32::max_value()`.
    pub fn slots_per_second(&self) -> Option<f64> {
        let speeds = self
            .0
            .windows(2)
            .filter_map(|windows| {
                let (slot_a, instant_a) = windows[0];
                let (slot_b, instant_b) = windows[1];

                // Taking advantage of saturating subtraction on `Slot`.
                let distance = f64::from((slot_b - slot_a).as_u64() as u32);

                let seconds = f64::from((instant_b - instant_a).as_millis() as u32) / 1_000.0;

                if seconds > 0.0 {
                    Some(distance / seconds)
                } else {
                    None
                }
            })
            .collect::<Vec<f64>>();

        let count = speeds.len();
        let sum: f64 = speeds.iter().sum();

        if count > 0 {
            Some(sum / f64::from(count as u32))
        } else {
            None
        }
    }

    /// Returns the time we should reach the given `slot`, judging by the latest observation and
    /// historical average syncing time.
    ///
    /// Returns `None` if the slot is prior to our latest observed slot or we have not made any
    /// observations.
    pub fn estimated_time_till_slot(&self, target_slot: Slot) -> Option<f64> {
        let (prev_slot, _) = self.0.last()?;
        let slots_per_second = self.slots_per_second()?;

        if target_slot > *prev_slot && slots_per_second > 0.0 {
            let distance = (target_slot - *prev_slot).as_u64() as f64;
            Some(distance / slots_per_second)
        } else {
            None
        }
    }
}
<|endoftext|>