<commit_before>use log_parser::*;
use timing_analyzer::Timing;
use error_analyzer::HttpErrorState;
use chrono::*;

#[derive(Clone, Debug, PartialEq)]
pub struct RequestResponsePair {
    pub request: log_events::Request,
    pub response: log_events::Response,
}

pub fn extract_matching_request_response_pairs(requests: &mut Vec<log_events::Request>,
                                               responses: &mut Vec<log_events::Response>)
                                               -> Vec<RequestResponsePair> {
    let mut request_response_pairs: Vec<RequestResponsePair> = Vec::new();

    for request_index in 0..requests.len() {
        {
            let request = requests.get(request_index);

            if request.is_none() {
                continue;
            }
        }

        let matching_response_index: Option<usize> = responses.iter()
            .position(|response| requests[request_index].id == response.id);

        if matching_response_index.is_some() {
            let request = requests.remove(request_index);
            let response = responses.remove(matching_response_index.unwrap());

            let pair = RequestResponsePair {
                request: request,
                response: response,
            };

            request_response_pairs.push(pair);
        }
    }

    request_response_pairs
}

impl Timing for RequestResponsePair {
    fn num_milliseconds(&self) -> i64 {
        self.response.response_time.num_milliseconds()
    }
}

impl Timing for Box<Timing> {
    fn num_milliseconds(&self) -> i64 {
        (**self).num_milliseconds()
    }
}

impl Timing for Box<RequestResponsePair> {
    fn num_milliseconds(&self) -> i64 {
        (**self).num_milliseconds()
    }
}

impl HttpErrorState for RequestResponsePair {
    fn error(&self) -> Option<log_events::HttpError> {
        self.response.http_error.clone()
    }
}

impl HttpErrorState for Box<RequestResponsePair> {
    fn error(&self) -> Option<log_events::HttpError> {
        (**self).error()
    }
}

pub struct RequestResponsePairIterator<'a> {
    events: &'a mut Iterator<Item = log_events::LogEvent>,
}

impl<'a> Iterator for RequestResponsePairIterator<'a> {
    type Item = RequestResponsePair;
    fn next(&mut self) -> Option<RequestResponsePair> {
        // self.events.next()
        Some(RequestResponsePair {
            request: log_events::Request {
                id: 1,
                time: DateTime::parse_from_str("08/Apr/2016:09:57:47 +0200",
                                               "%d/%b/%Y:%H:%M:%S %z")
                    .unwrap(),
                original_log_line: "whatever".to_string(),
            },
            response: log_events::Response {
                id: 1,
                response_time: Duration::milliseconds(7),
                original_log_line: "whatever".to_string(),
                http_error: None,
            },
        })
    }
}

// fn extract_matching_request_response_pairs_iter<T, U>(events: T) -> U
//     where T: Iterator<Item = log_events::LogEvent>,
//           U: Iterator<Item = RequestResponsePair>
// {
//     vec![RequestResponsePair {
//              request: log_events::Request {
//                  id: 1,
//                  time: DateTime::parse_from_str("08/Apr/2016:09:57:47 +0200",
//                                                 "%d/%b/%Y:%H:%M:%S %z")
//                      .unwrap(),
//                  original_log_line: "whatever".to_string(),
//              },
//              response: log_events::Response {
//                  id: 1,
//                  response_time: Duration::milliseconds(7),
//                  original_log_line: "whatever".to_string(),
//                  http_error: None,
//              },
//          }]
//         .iter() as U
// }

#[cfg(test)]
mod tests {
    use chrono::*;
    use super::*;
    use log_parser;
    use timing_analyzer::Timing;
    use error_analyzer::HttpErrorState;

    #[test]
    fn test_extract_matching_request_response_pairs() {
        let mut requests = vec![log_parser::log_events::Request {
                                    id: 1,
                                    time: DateTime::parse_from_str("08/Apr/2016:09:57:47 +0200",
                                                                   "%d/%b/%Y:%H:%M:%S %z")
                                        .unwrap(),
                                    original_log_line: "whatever".to_string(),
                                },
                                log_parser::log_events::Request {
                                    id: 77,
                                    time: DateTime::parse_from_str("08/Apr/2016:09:57:47 +0200",
                                                                   "%d/%b/%Y:%H:%M:%S %z")
                                        .unwrap(),
                                    original_log_line: "whatever".to_string(),
                                }];

        let mut responses = vec![log_parser::log_events::Response {
                                     id: 1,
                                     response_time: Duration::milliseconds(7),
                                     original_log_line: "whatever".to_string(),
                                     http_error: None,
                                 },
                                 log_parser::log_events::Response {
                                     id: 99,
                                     response_time: Duration::milliseconds(10),
                                     original_log_line: "whatever".to_string(),
                                     http_error: None,
                                 }];

        let result = extract_matching_request_response_pairs(&mut requests, &mut responses);

        assert_eq!(result.len(), 1);
        assert_eq!(result[0].request.id, 1);
        assert_eq!(result[0].response.id, 1);

        assert_eq!(requests.len(), 1);
        assert_eq!(responses.len(), 1);
    }

    #[test]
    fn test_extract_matching_request_response_pairs_iterator() {
        let mut events =
            vec![log_parser::log_events::LogEvent::Request(log_parser::log_events::Request {
                     id: 1,
                     time: DateTime::parse_from_str("08/Apr/2016:09:57:47 +0200",
                                                    "%d/%b/%Y:%H:%M:%S %z")
                         .unwrap(),
                     original_log_line: "whatever".to_string(),
                 }),
                 log_parser::log_events::LogEvent::Response(log_parser::log_events::Response {
                     id: 1,
                     response_time: Duration::milliseconds(7),
                     original_log_line: "whatever".to_string(),
                     http_error: None,
                 })];

        // // let result = extract_matching_request_response_pairs(&mut requests, &mut responses);
        // let result = extract_matching_request_response_pairs_iter(events.iter()).collect();
        //
        // assert_eq!(result.len(), 1);
        // assert_eq!(result[0].request.id, 1);
        // assert_eq!(result[0].response.id, 1);

        let mut iterator = RequestResponsePairIterator {
            events: &mut events.into_iter() as
                    &mut Iterator<Item = log_parser::log_events::LogEvent>,
        };

        assert_eq!(iterator.next(), None);
    }

    #[test]
    fn test_timing_trait() {
        let timing: &Timing = &RequestResponsePair {
            request: log_parser::log_events::Request {
                id: 1,
                time: DateTime::parse_from_str("08/Apr/2016:09:57:47 +0200",
                                               "%d/%b/%Y:%H:%M:%S %z")
                    .unwrap(),
                original_log_line: "whatever".to_string(),
            },
            response: log_parser::log_events::Response {
                id: 1,
                response_time: Duration::milliseconds(7),
                original_log_line: "whatever".to_string(),
                http_error: None,
            },
        } as &Timing;

        let result: i64 = timing.num_milliseconds();
        assert_eq!(result, 7);

        let boxed_timing = Box::new(timing);

        let result: i64 = boxed_timing.num_milliseconds();
        assert_eq!(result, 7);
    }

    #[test]
    fn test_http_error_state_trait() {
        let state: &HttpErrorState = &RequestResponsePair {
            request: log_parser::log_events::Request {
                id: 1,
                time: DateTime::parse_from_str("08/Apr/2016:09:57:47 +0200",
                                               "%d/%b/%Y:%H:%M:%S %z")
                    .unwrap(),
                original_log_line: "whatever".to_string(),
            },
            response: log_parser::log_events::Response {
                id: 1,
                response_time: Duration::milliseconds(7),
                original_log_line: "whatever".to_string(),
                http_error: None,
            },
        } as &HttpErrorState;

        let result = state.error();
        assert_eq!(result, None);

        let boxed_state = Box::new(state);

        let result = boxed_state.error();
        assert_eq!(result, None);
    }
}
<commit_msg>Working prototyp of Events in, Pairs out<commit_after>use log_parser::*;
use timing_analyzer::Timing;
use error_analyzer::HttpErrorState;

#[derive(Clone, Debug, PartialEq)]
pub struct RequestResponsePair {
    pub request: log_events::Request,
    pub response: log_events::Response,
}

pub fn extract_matching_request_response_pairs(requests: &mut Vec<log_events::Request>,
                                               responses: &mut Vec<log_events::Response>)
                                               -> Vec<RequestResponsePair> {
    let mut request_response_pairs: Vec<RequestResponsePair> = Vec::new();

    for request_index in 0..requests.len() {
        {
            let request = requests.get(request_index);

            if request.is_none() {
                continue;
            }
        }

        let matching_response_index: Option<usize> = responses.iter()
            .position(|response| requests[request_index].id == response.id);

        if matching_response_index.is_some() {
            let request = requests.remove(request_index);
            let response = responses.remove(matching_response_index.unwrap());

            let pair = RequestResponsePair {
                request: request,
                response: response,
            };

            request_response_pairs.push(pair);
        }
    }

    request_response_pairs
}

impl Timing for RequestResponsePair {
    fn num_milliseconds(&self) -> i64 {
        self.response.response_time.num_milliseconds()
    }
}

impl Timing for Box<Timing> {
    fn num_milliseconds(&self) -> i64 {
        (**self).num_milliseconds()
    }
}

impl Timing for Box<RequestResponsePair> {
    fn num_milliseconds(&self) -> i64 {
        (**self).num_milliseconds()
    }
}

impl HttpErrorState for RequestResponsePair {
    fn error(&self) -> Option<log_events::HttpError> {
        self.response.http_error.clone()
    }
}

impl HttpErrorState for Box<RequestResponsePair> {
    fn error(&self) -> Option<log_events::HttpError> {
        (**self).error()
    }
}

pub struct RequestResponsePairIterator<'a> {
    events: &'a mut Iterator<Item = log_events::LogEvent>,
    requests: Vec<log_events::Request>,
    responses: Vec<log_events::Response>,
}

impl<'a> Iterator for RequestResponsePairIterator<'a> {
    type Item = RequestResponsePair;

    fn next(&mut self) -> Option<RequestResponsePair> {
        let mut pair = None;

        while pair.is_none() {
            let event = self.events.next();

            match event {
                Some(log_events::LogEvent::Request(request)) => self.requests.push(request),
                Some(log_events::LogEvent::Response(response)) => self.responses.push(response),
                None => return None,
            }

            pair = extract_first_matching_request_response_pair(&mut self.requests,
                                                                &mut self.responses);
        }
        pair
    }
}

pub fn extract_first_matching_request_response_pair(requests: &mut Vec<log_events::Request>,
                                                    responses: &mut Vec<log_events::Response>)
                                                    -> Option<RequestResponsePair> {
    for request_index in 0..requests.len() {
        {
            let request = requests.get(request_index);

            if request.is_none() {
                continue;
            }
        }

        let matching_response_index: Option<usize> = responses.iter()
            .position(|response| requests[request_index].id == response.id);

        if matching_response_index.is_some() {
            let request = requests.remove(request_index);
            let response = responses.remove(matching_response_index.unwrap());

            let pair = RequestResponsePair {
                request: request,
                response: response,
            };

            return Some(pair);
        }
    }
    None
}

#[cfg(test)]
mod tests {
    use chrono::*;
    use super::*;
    use log_parser;
    use timing_analyzer::Timing;
    use error_analyzer::HttpErrorState;

    #[test]
    fn test_extract_matching_request_response_pairs() {
        let mut requests = vec![log_parser::log_events::Request {
                                    id: 1,
                                    time: DateTime::parse_from_str("08/Apr/2016:09:57:47 +0200",
                                                                   "%d/%b/%Y:%H:%M:%S %z")
                                        .unwrap(),
                                    original_log_line: "whatever".to_string(),
                                },
                                log_parser::log_events::Request {
                                    id: 77,
                                    time: DateTime::parse_from_str("08/Apr/2016:09:57:47 +0200",
                                                                   "%d/%b/%Y:%H:%M:%S %z")
                                        .unwrap(),
                                    original_log_line: "whatever".to_string(),
                                }];

        let mut responses = vec![log_parser::log_events::Response {
                                     id: 1,
                                     response_time: Duration::milliseconds(7),
                                     original_log_line: "whatever".to_string(),
                                     http_error: None,
                                 },
                                 log_parser::log_events::Response {
                                     id: 99,
                                     response_time: Duration::milliseconds(10),
                                     original_log_line: "whatever".to_string(),
                                     http_error: None,
                                 }];

        let result = extract_matching_request_response_pairs(&mut requests, &mut responses);

        assert_eq!(result.len(), 1);
        assert_eq!(result[0].request.id, 1);
        assert_eq!(result[0].response.id, 1);

        assert_eq!(requests.len(), 1);
        assert_eq!(responses.len(), 1);
    }

    #[test]
    fn test_extract_matching_request_response_pairs_iterator() {
        let events =
            vec![log_parser::log_events::LogEvent::Request(log_parser::log_events::Request {
                     id: 1,
                     time: DateTime::parse_from_str("08/Apr/2016:09:57:47 +0200",
                                                    "%d/%b/%Y:%H:%M:%S %z")
                         .unwrap(),
                     original_log_line: "whatever".to_string(),
                 }),
                 log_parser::log_events::LogEvent::Response(log_parser::log_events::Response {
                     id: 1,
                     response_time: Duration::milliseconds(7),
                     original_log_line: "whatever".to_string(),
                     http_error: None,
                 })];

        let mut iterator = RequestResponsePairIterator {
            events: &mut events.into_iter() as
                    &mut Iterator<Item = log_parser::log_events::LogEvent>,
            requests: vec![],
            responses: vec![],
        };

        let result = iterator.next().unwrap();
        assert_eq!(result.request.id, 1);
    }

    #[test]
    fn test_timing_trait() {
        let timing: &Timing = &RequestResponsePair {
            request: log_parser::log_events::Request {
                id: 1,
                time: DateTime::parse_from_str("08/Apr/2016:09:57:47 +0200",
                                               "%d/%b/%Y:%H:%M:%S %z")
                    .unwrap(),
                original_log_line: "whatever".to_string(),
            },
            response: log_parser::log_events::Response {
                id: 1,
                response_time: Duration::milliseconds(7),
                original_log_line: "whatever".to_string(),
                http_error: None,
            },
        } as &Timing;

        let result: i64 = timing.num_milliseconds();
        assert_eq!(result, 7);

        let boxed_timing = Box::new(timing);

        let result: i64 = boxed_timing.num_milliseconds();
        assert_eq!(result, 7);
    }

    #[test]
    fn test_http_error_state_trait() {
        let state: &HttpErrorState = &RequestResponsePair {
            request: log_parser::log_events::Request {
                id: 1,
                time: DateTime::parse_from_str("08/Apr/2016:09:57:47 +0200",
                                               "%d/%b/%Y:%H:%M:%S %z")
                    .unwrap(),
                original_log_line: "whatever".to_string(),
            },
            response: log_parser::log_events::Response {
                id: 1,
                response_time: Duration::milliseconds(7),
                original_log_line: "whatever".to_string(),
                http_error: None,
            },
        } as &HttpErrorState;

        let result = state.error();
        assert_eq!(result, None);

        let boxed_state = Box::new(state);

        let result = boxed_state.error();
        assert_eq!(result, None);
    }
}
<|endoftext|>


<commit_before>/* Find ECB encrypted string
 *
 * Dmitry Vasiliev <dima@hlabs.org>
 */

extern mod extra;

use std::hashmap::HashSet;
#[cfg(not(test))]
use std::path::Path;
#[cfg(not(test))]
use std::io::fs::File;
#[cfg(not(test))]
use std::io::buffered::BufferedReader;

#[cfg(not(test))]
use extra::hex::FromHex;

#[cfg(not(test))]
enum ECBEncryptedLine {
    Found(uint, ~str),
    NotFound
}

/*
 * Find 128 bit ECB encrypted line in a file
 */
#[cfg(not(test))]
fn find_ecb_encrypted_line(file: File) -> ECBEncryptedLine {

    let mut reader = BufferedReader::new(file);
    for (line_num, line) in reader.lines().enumerate() {
        if is_buffer_ecb_encrypted(line.from_hex().unwrap()) {
            // Return the first found line.
            // Also convert 0-based line number to 1-based
            return Found(line_num + 1, line);
        }
    }
    NotFound
}

/*
 * Is buffer ECB encrypted?
 */
#[inline]
fn is_buffer_ecb_encrypted(buffer: &[u8]) -> bool {
    // 128 bit encryption
    static ecb_block_size: uint = 16;

    let mut blocks = HashSet::new();
    for block in buffer.chunks(ecb_block_size) {
        if !blocks.insert(block) {
            return true;
        }
    }
    false
}

/*
 * Main entry point
 */
#[cfg(not(test))]
fn main() {
    let path = Path::new("ciphertexts.txt");
    let result = match File::open(&path) {
        Some(file) => find_ecb_encrypted_line(file),
        None => fail!("Unable to open ciphertexts.txt")
    };
    match result {
        Found(line_num, text) => println!("Found ECB encrypted text at \
                                           line {}: {}", line_num, text),
        NotFound => println("No ECB encrypted text found")
    }
}

/*
 * Tests
 */
#[cfg(test)]
mod test {
    use super::is_buffer_ecb_encrypted;
    use extra::hex::FromHex;

    #[test]
    fn test_is_buffer_ecb_encrypted() {
        let buffer = ~"000102030405060708090a0b0c0d0e0f\
                       101112131415161718191a1b1c1d1e1f\
                       202122232425262728292a2b2c2d2e2f";
        assert_eq!(is_buffer_ecb_encrypted(buffer.from_hex().unwrap()), false);

        let buffer = ~"000102030405060708090a0b0c0d0e0f\
                       101112131415161718191a1b1c1d1e1f\
                       000102030405060708090a0b0c0d0e0f";
        assert_eq!(is_buffer_ecb_encrypted(buffer.from_hex().unwrap()), true);
    }
}
<commit_msg>Replace println() with println!()<commit_after>/* Find ECB encrypted string
 *
 * Dmitry Vasiliev <dima@hlabs.org>
 */

extern mod extra;

use std::hashmap::HashSet;
#[cfg(not(test))]
use std::path::Path;
#[cfg(not(test))]
use std::io::fs::File;
#[cfg(not(test))]
use std::io::buffered::BufferedReader;

#[cfg(not(test))]
use extra::hex::FromHex;

#[cfg(not(test))]
enum ECBEncryptedLine {
    Found(uint, ~str),
    NotFound
}

/*
 * Find 128 bit ECB encrypted line in a file
 */
#[cfg(not(test))]
fn find_ecb_encrypted_line(file: File) -> ECBEncryptedLine {

    let mut reader = BufferedReader::new(file);
    for (line_num, line) in reader.lines().enumerate() {
        if is_buffer_ecb_encrypted(line.from_hex().unwrap()) {
            // Return the first found line.
            // Also convert 0-based line number to 1-based
            return Found(line_num + 1, line);
        }
    }
    NotFound
}

/*
 * Is buffer ECB encrypted?
 */
#[inline]
fn is_buffer_ecb_encrypted(buffer: &[u8]) -> bool {
    // 128 bit encryption
    static ecb_block_size: uint = 16;

    let mut blocks = HashSet::new();
    for block in buffer.chunks(ecb_block_size) {
        if !blocks.insert(block) {
            return true;
        }
    }
    false
}

/*
 * Main entry point
 */
#[cfg(not(test))]
fn main() {
    let path = Path::new("ciphertexts.txt");
    let result = match File::open(&path) {
        Some(file) => find_ecb_encrypted_line(file),
        None => fail!("Unable to open ciphertexts.txt")
    };
    match result {
        Found(line_num, text) => println!("Found ECB encrypted text at \
                                           line {}: {}", line_num, text),
        NotFound => println!("No ECB encrypted text found")
    }
}

/*
 * Tests
 */
#[cfg(test)]
mod test {
    use super::is_buffer_ecb_encrypted;
    use extra::hex::FromHex;

    #[test]
    fn test_is_buffer_ecb_encrypted() {
        let buffer = ~"000102030405060708090a0b0c0d0e0f\
                       101112131415161718191a1b1c1d1e1f\
                       202122232425262728292a2b2c2d2e2f";
        assert_eq!(is_buffer_ecb_encrypted(buffer.from_hex().unwrap()), false);

        let buffer = ~"000102030405060708090a0b0c0d0e0f\
                       101112131415161718191a1b1c1d1e1f\
                       000102030405060708090a0b0c0d0e0f";
        assert_eq!(is_buffer_ecb_encrypted(buffer.from_hex().unwrap()), true);
    }
}
<|endoftext|>


<commit_before>use clap::ArgMatches;
use enr::Enr;
use libp2p::gossipsub::{GossipsubConfig, GossipsubConfigBuilder};
use serde_derive::{Deserialize, Serialize};
use std::path::PathBuf;
use std::time::Duration;

/// The beacon node topic string to subscribe to.
pub const BEACON_PUBSUB_TOPIC: &str = "beacon_block";
pub const BEACON_ATTESTATION_TOPIC: &str = "beacon_attestation";
//TODO: Implement shard subnets
pub const SHARD_TOPIC_PREFIX: &str = "shard";

#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
/// Network configuration for lighthouse.
pub struct Config {
    /// Data directory where node's keyfile is stored
    pub network_dir: PathBuf,

    /// IP address to listen on.
    pub listen_address: std::net::IpAddr,

    /// The TCP port that libp2p listens on.
    pub libp2p_port: u16,

    /// The address to broadcast to peers about which address we are listening on.
    pub discovery_address: std::net::IpAddr,

    /// UDP port that discovery listens on.
    pub discovery_port: u16,

    /// Target number of connected peers.
    pub max_peers: usize,

    /// Gossipsub configuration parameters.
    #[serde(skip)]
    pub gs_config: GossipsubConfig,

    /// List of nodes to initially connect to.
    pub boot_nodes: Vec<Enr>,

    /// Client version
    pub client_version: String,

    /// List of extra topics to initially subscribe to as strings.
    pub topics: Vec<String>,
}

impl Default for Config {
    /// Generate a default network configuration.
    fn default() -> Self {
        let mut network_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from("."));
        network_dir.push(".lighthouse");
        network_dir.push("network");
        Config {
            network_dir,
            listen_address: "127.0.0.1".parse().expect("vaild ip address"),
            libp2p_port: 9000,
            discovery_address: "127.0.0.1".parse().expect("valid ip address"),
            discovery_port: 9000,
            max_peers: 10,
            //TODO: Set realistic values for production
            gs_config: GossipsubConfigBuilder::new()
                .max_gossip_size(4_000_000)
                .inactivity_timeout(Duration::from_secs(90))
                .heartbeat_interval(Duration::from_secs(20))
                .build(),
            boot_nodes: vec![],
            client_version: version::version(),
            topics: Vec::new(),
        }
    }
}

/// Generates a default Config.
impl Config {
    pub fn new() -> Self {
        Config::default()
    }

    pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), String> {
        dbg!(self.network_dir.clone());
        if let Some(dir) = args.value_of("datadir") {
            self.network_dir = PathBuf::from(dir).join("network");
        };
        dbg!(self.network_dir.clone());

        if let Some(listen_address_str) = args.value_of("listen-address") {
            let listen_address = listen_address_str
                .parse()
                .map_err(|_| format!("Invalid listen address: {:?}", listen_address_str))?;
            self.listen_address = listen_address;
            self.discovery_address = listen_address;
        }

        if let Some(max_peers_str) = args.value_of("maxpeers") {
            self.max_peers = max_peers_str
                .parse::<usize>()
                .map_err(|_| format!("Invalid number of max peers: {}", max_peers_str))?;
        }

        if let Some(port_str) = args.value_of("port") {
            let port = port_str
                .parse::<u16>()
                .map_err(|_| format!("Invalid port: {}", port_str))?;
            self.libp2p_port = port;
            self.discovery_port = port;
        }

        if let Some(boot_enr_str) = args.value_of("boot-nodes") {
            self.boot_nodes = boot_enr_str
                .split(',')
                .map(|enr| enr.parse().map_err(|_| format!("Invalid ENR: {}", enr)))
                .collect::<Result<Vec<Enr>, _>>()?;
        }

        if let Some(discovery_address_str) = args.value_of("discovery-address") {
            self.discovery_address = discovery_address_str
                .parse()
                .map_err(|_| format!("Invalid discovery address: {:?}", discovery_address_str))?
        }

        if let Some(disc_port_str) = args.value_of("disc-port") {
            self.discovery_port = disc_port_str
                .parse::<u16>()
                .map_err(|_| format!("Invalid discovery port: {}", disc_port_str))?;
        }

        Ok(())
    }
}
<commit_msg>Removes left-over debugging statements<commit_after>use clap::ArgMatches;
use enr::Enr;
use libp2p::gossipsub::{GossipsubConfig, GossipsubConfigBuilder};
use serde_derive::{Deserialize, Serialize};
use std::path::PathBuf;
use std::time::Duration;

/// The beacon node topic string to subscribe to.
pub const BEACON_PUBSUB_TOPIC: &str = "beacon_block";
pub const BEACON_ATTESTATION_TOPIC: &str = "beacon_attestation";
//TODO: Implement shard subnets
pub const SHARD_TOPIC_PREFIX: &str = "shard";

#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
/// Network configuration for lighthouse.
pub struct Config {
    /// Data directory where node's keyfile is stored
    pub network_dir: PathBuf,

    /// IP address to listen on.
    pub listen_address: std::net::IpAddr,

    /// The TCP port that libp2p listens on.
    pub libp2p_port: u16,

    /// The address to broadcast to peers about which address we are listening on.
    pub discovery_address: std::net::IpAddr,

    /// UDP port that discovery listens on.
    pub discovery_port: u16,

    /// Target number of connected peers.
    pub max_peers: usize,

    /// Gossipsub configuration parameters.
    #[serde(skip)]
    pub gs_config: GossipsubConfig,

    /// List of nodes to initially connect to.
    pub boot_nodes: Vec<Enr>,

    /// Client version
    pub client_version: String,

    /// List of extra topics to initially subscribe to as strings.
    pub topics: Vec<String>,
}

impl Default for Config {
    /// Generate a default network configuration.
    fn default() -> Self {
        let mut network_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from("."));
        network_dir.push(".lighthouse");
        network_dir.push("network");
        Config {
            network_dir,
            listen_address: "127.0.0.1".parse().expect("vaild ip address"),
            libp2p_port: 9000,
            discovery_address: "127.0.0.1".parse().expect("valid ip address"),
            discovery_port: 9000,
            max_peers: 10,
            //TODO: Set realistic values for production
            gs_config: GossipsubConfigBuilder::new()
                .max_gossip_size(4_000_000)
                .inactivity_timeout(Duration::from_secs(90))
                .heartbeat_interval(Duration::from_secs(20))
                .build(),
            boot_nodes: vec![],
            client_version: version::version(),
            topics: Vec::new(),
        }
    }
}

/// Generates a default Config.
impl Config {
    pub fn new() -> Self {
        Config::default()
    }

    pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), String> {
        if let Some(dir) = args.value_of("datadir") {
            self.network_dir = PathBuf::from(dir).join("network");
        };

        if let Some(listen_address_str) = args.value_of("listen-address") {
            let listen_address = listen_address_str
                .parse()
                .map_err(|_| format!("Invalid listen address: {:?}", listen_address_str))?;
            self.listen_address = listen_address;
            self.discovery_address = listen_address;
        }

        if let Some(max_peers_str) = args.value_of("maxpeers") {
            self.max_peers = max_peers_str
                .parse::<usize>()
                .map_err(|_| format!("Invalid number of max peers: {}", max_peers_str))?;
        }

        if let Some(port_str) = args.value_of("port") {
            let port = port_str
                .parse::<u16>()
                .map_err(|_| format!("Invalid port: {}", port_str))?;
            self.libp2p_port = port;
            self.discovery_port = port;
        }

        if let Some(boot_enr_str) = args.value_of("boot-nodes") {
            self.boot_nodes = boot_enr_str
                .split(',')
                .map(|enr| enr.parse().map_err(|_| format!("Invalid ENR: {}", enr)))
                .collect::<Result<Vec<Enr>, _>>()?;
        }

        if let Some(discovery_address_str) = args.value_of("discovery-address") {
            self.discovery_address = discovery_address_str
                .parse()
                .map_err(|_| format!("Invalid discovery address: {:?}", discovery_address_str))?
        }

        if let Some(disc_port_str) = args.value_of("disc-port") {
            self.discovery_port = disc_port_str
                .parse::<u16>()
                .map_err(|_| format!("Invalid discovery port: {}", disc_port_str))?;
        }

        Ok(())
    }
}
<|endoftext|>


<commit_before>use crate::types::GossipKind;
use crate::{Enr, PeerIdSerialized};
use discv5::{Discv5Config, Discv5ConfigBuilder};
use libp2p::gossipsub::{
    GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage, MessageId, ValidationMode,
};
use libp2p::Multiaddr;
use serde_derive::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::path::PathBuf;
use std::time::Duration;

pub const GOSSIP_MAX_SIZE: usize = 1_048_576;

#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
/// Network configuration for lighthouse.
pub struct Config {
    /// Data directory where node's keyfile is stored
    pub network_dir: PathBuf,

    /// IP address to listen on.
    pub listen_address: std::net::IpAddr,

    /// The TCP port that libp2p listens on.
    pub libp2p_port: u16,

    /// UDP port that discovery listens on.
    pub discovery_port: u16,

    /// The address to broadcast to peers about which address we are listening on. None indicates
    /// that no discovery address has been set in the CLI args.
    pub enr_address: Option<std::net::IpAddr>,

    /// The udp port to broadcast to peers in order to reach back for discovery.
    pub enr_udp_port: Option<u16>,

    /// The tcp port to broadcast to peers in order to reach back for libp2p services.
    pub enr_tcp_port: Option<u16>,

    /// Target number of connected peers.
    pub target_peers: usize,

    /// Gossipsub configuration parameters.
    #[serde(skip)]
    pub gs_config: GossipsubConfig,

    /// Discv5 configuration parameters.
    #[serde(skip)]
    pub discv5_config: Discv5Config,

    /// List of nodes to initially connect to.
    pub boot_nodes_enr: Vec<Enr>,

    /// List of nodes to initially connect to, on Multiaddr format.
    pub boot_nodes_multiaddr: Vec<Multiaddr>,

    /// List of libp2p nodes to initially connect to.
    pub libp2p_nodes: Vec<Multiaddr>,

    /// List of trusted libp2p nodes which are not scored.
    pub trusted_peers: Vec<PeerIdSerialized>,

    /// Client version
    pub client_version: String,

    /// Disables the discovery protocol from starting.
    pub disable_discovery: bool,

    /// Attempt to construct external port mappings with UPnP.
    pub upnp_enabled: bool,

    /// List of extra topics to initially subscribe to as strings.
    pub topics: Vec<GossipKind>,
}

impl Default for Config {
    /// Generate a default network configuration.
    fn default() -> Self {
        let mut network_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from("."));
        network_dir.push(".lighthouse");
        network_dir.push("network");

        // The function used to generate a gossipsub message id
        // We use the first 8 bytes of SHA256(data) for content addressing
        let gossip_message_id =
            |message: &GossipsubMessage| MessageId::from(&Sha256::digest(&message.data)[..8]);

        // gossipsub configuration
        // Note: The topics by default are sent as plain strings. Hashes are an optional
        // parameter.
        let gs_config = GossipsubConfigBuilder::new()
            .max_transmit_size(GOSSIP_MAX_SIZE)
            .heartbeat_interval(Duration::from_millis(700))
            .mesh_n(8)
            .mesh_n_low(6)
            .mesh_n_high(12)
            .gossip_lazy(6)
            .fanout_ttl(Duration::from_secs(60))
            .history_length(6)
            .history_gossip(3)
            .validate_messages() // require validation before propagation
            .validation_mode(ValidationMode::Anonymous)
            // prevent duplicates for 550 heartbeats(700millis * 550) = 385 secs
            .duplicate_cache_time(Duration::from_secs(385))
            .message_id_fn(gossip_message_id)
            .build()
            .expect("valid gossipsub configuration");

        // discv5 configuration
        let discv5_config = Discv5ConfigBuilder::new()
            .enable_packet_filter()
            .session_cache_capacity(1000)
            .request_timeout(Duration::from_secs(4))
            .request_retries(1)
            .enr_peer_update_min(10)
            .query_parallelism(5)
            .query_timeout(Duration::from_secs(30))
            .query_peer_timeout(Duration::from_secs(2))
            .ip_limit() // limits /24 IP's in buckets.
            .ping_interval(Duration::from_secs(300))
            .build();

        // NOTE: Some of these get overridden by the corresponding CLI default values.
        Config {
            network_dir,
            listen_address: "0.0.0.0".parse().expect("valid ip address"),
            libp2p_port: 9000,
            discovery_port: 9000,
            enr_address: None,
            enr_udp_port: None,
            enr_tcp_port: None,
            target_peers: 50,
            gs_config,
            discv5_config,
            boot_nodes_enr: vec![],
            boot_nodes_multiaddr: vec![],
            libp2p_nodes: vec![],
            trusted_peers: vec![],
            client_version: lighthouse_version::version_with_platform(),
            disable_discovery: false,
            upnp_enabled: true,
            topics: Vec::new(),
        }
    }
}
<commit_msg>Increase content-id length (#1725)<commit_after>use crate::types::GossipKind;
use crate::{Enr, PeerIdSerialized};
use discv5::{Discv5Config, Discv5ConfigBuilder};
use libp2p::gossipsub::{
    GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage, MessageId, ValidationMode,
};
use libp2p::Multiaddr;
use serde_derive::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::path::PathBuf;
use std::time::Duration;

pub const GOSSIP_MAX_SIZE: usize = 1_048_576;

#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
/// Network configuration for lighthouse.
pub struct Config {
    /// Data directory where node's keyfile is stored
    pub network_dir: PathBuf,

    /// IP address to listen on.
    pub listen_address: std::net::IpAddr,

    /// The TCP port that libp2p listens on.
    pub libp2p_port: u16,

    /// UDP port that discovery listens on.
    pub discovery_port: u16,

    /// The address to broadcast to peers about which address we are listening on. None indicates
    /// that no discovery address has been set in the CLI args.
    pub enr_address: Option<std::net::IpAddr>,

    /// The udp port to broadcast to peers in order to reach back for discovery.
    pub enr_udp_port: Option<u16>,

    /// The tcp port to broadcast to peers in order to reach back for libp2p services.
    pub enr_tcp_port: Option<u16>,

    /// Target number of connected peers.
    pub target_peers: usize,

    /// Gossipsub configuration parameters.
    #[serde(skip)]
    pub gs_config: GossipsubConfig,

    /// Discv5 configuration parameters.
    #[serde(skip)]
    pub discv5_config: Discv5Config,

    /// List of nodes to initially connect to.
    pub boot_nodes_enr: Vec<Enr>,

    /// List of nodes to initially connect to, on Multiaddr format.
    pub boot_nodes_multiaddr: Vec<Multiaddr>,

    /// List of libp2p nodes to initially connect to.
    pub libp2p_nodes: Vec<Multiaddr>,

    /// List of trusted libp2p nodes which are not scored.
    pub trusted_peers: Vec<PeerIdSerialized>,

    /// Client version
    pub client_version: String,

    /// Disables the discovery protocol from starting.
    pub disable_discovery: bool,

    /// Attempt to construct external port mappings with UPnP.
    pub upnp_enabled: bool,

    /// List of extra topics to initially subscribe to as strings.
    pub topics: Vec<GossipKind>,
}

impl Default for Config {
    /// Generate a default network configuration.
    fn default() -> Self {
        let mut network_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from("."));
        network_dir.push(".lighthouse");
        network_dir.push("network");

        // The function used to generate a gossipsub message id
        // We use the first 8 bytes of SHA256(data) for content addressing
        let gossip_message_id =
            |message: &GossipsubMessage| MessageId::from(&Sha256::digest(&message.data)[..]);

        // gossipsub configuration
        // Note: The topics by default are sent as plain strings. Hashes are an optional
        // parameter.
        let gs_config = GossipsubConfigBuilder::new()
            .max_transmit_size(GOSSIP_MAX_SIZE)
            .heartbeat_interval(Duration::from_millis(700))
            .mesh_n(8)
            .mesh_n_low(6)
            .mesh_n_high(12)
            .gossip_lazy(6)
            .fanout_ttl(Duration::from_secs(60))
            .history_length(6)
            .history_gossip(3)
            .validate_messages() // require validation before propagation
            .validation_mode(ValidationMode::Anonymous)
            // prevent duplicates for 550 heartbeats(700millis * 550) = 385 secs
            .duplicate_cache_time(Duration::from_secs(385))
            .message_id_fn(gossip_message_id)
            .build()
            .expect("valid gossipsub configuration");

        // discv5 configuration
        let discv5_config = Discv5ConfigBuilder::new()
            .enable_packet_filter()
            .session_cache_capacity(1000)
            .request_timeout(Duration::from_secs(4))
            .request_retries(1)
            .enr_peer_update_min(10)
            .query_parallelism(5)
            .query_timeout(Duration::from_secs(30))
            .query_peer_timeout(Duration::from_secs(2))
            .ip_limit() // limits /24 IP's in buckets.
            .ping_interval(Duration::from_secs(300))
            .build();

        // NOTE: Some of these get overridden by the corresponding CLI default values.
        Config {
            network_dir,
            listen_address: "0.0.0.0".parse().expect("valid ip address"),
            libp2p_port: 9000,
            discovery_port: 9000,
            enr_address: None,
            enr_udp_port: None,
            enr_tcp_port: None,
            target_peers: 50,
            gs_config,
            discv5_config,
            boot_nodes_enr: vec![],
            boot_nodes_multiaddr: vec![],
            libp2p_nodes: vec![],
            trusted_peers: vec![],
            client_version: lighthouse_version::version_with_platform(),
            disable_discovery: false,
            upnp_enabled: true,
            topics: Vec::new(),
        }
    }
}
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io
use super::model;
use regex;
use rs_es;
use rs_es::query::Query as rs_q;
use rs_es::operations::search::SearchResult;
use rs_es::units as rs_u;
use mimir;
use serde_json;
use serde;

fn build_rs_client(cnx: &String) -> rs_es::Client {
    let re = regex::Regex::new(r"(?:https?://)?(?P<host>.+?):(?P<port>\d+)").unwrap();
    let cap = re.captures(&cnx).unwrap();
    let host = cap.name("host").unwrap().as_str();
    let port = cap.name("port").unwrap().as_str().parse::<u32>().unwrap();

    rs_es::Client::new(&host, port)
}

/// takes a ES json blob and build a Place from it
/// it uses the _type field of ES to know which type of the Place enum to fill
pub fn make_place(doc_type: String, value: Option<Box<serde_json::Value>>) -> Option<mimir::Place> {
    value.and_then(|v| {
        fn convert<T: serde::Deserialize>(v: serde_json::Value,
                                          f: fn(T) -> mimir::Place)
                                          -> Option<mimir::Place> {
            serde_json::from_value::<T>(v)
                .map_err(|err| warn!("Impossible to load ES result: {}", err))
                .ok()
                .map(f)
        }
        match doc_type.as_ref() {
            "addr" => convert(*v, mimir::Place::Addr),
            "street" => convert(*v, mimir::Place::Street),
            "admin" => convert(*v, mimir::Place::Admin),
            "poi" => convert(*v, mimir::Place::Poi),
            "stop" => convert(*v, mimir::Place::Stop),
            _ => {
                warn!("unknown ES return value, _type field = {}", doc_type);
                None
            }
        }
    })
}

#[derive(Debug, Eq, PartialEq)]
enum MatchType {
    Prefix,
    Fuzzy,
}

fn build_query(q: &str,
               match_type: MatchType,
               coord: &Option<model::Coord>,
               shape: Option<Vec<rs_es::units::Location>>)
               -> rs_es::query::Query {
    use rs_es::query::functions::Function;
    // we order the type of object we want
    // Note: the addresses are boosted more because even if we don't want them first
    // because they are more severely filtered
    let boost_addr = rs_q::build_term("_type", "addr").with_boost(5000).build();
    let boost_admin = rs_q::build_term("_type", "admin").with_boost(3000).build();
    let boost_stop = rs_q::build_term("_type", "stop").with_boost(2000).build();

    let main_match_type = match match_type {
        MatchType::Prefix => "label.prefix",
        MatchType::Fuzzy => "label.ngram",
    };

    let boost_main_match_query = rs_q::build_multi_match(vec![main_match_type.to_string()],
                                                         q.to_string())
        .with_boost(100)
        .build();
    let boost_zipcode_match_query = rs_q::build_multi_match(vec!["zip_codes.prefix".to_string()],
                                                            q.to_string())
        .with_boost(500)
        .build();

    let mut should_query = vec![boost_addr,
                                boost_admin,
                                boost_stop,
                                boost_main_match_query,
                                boost_zipcode_match_query];

    // for fuzzy search we also search by the prefix index (with a greater boost than ngram)
    // to have better results
    if match_type == MatchType::Fuzzy {
        should_query.push(rs_q::build_match("label.prefix", q.to_string())
            .with_boost(1000)
            .build());
    }

    if let &Some(ref c) = coord {
        // if we have coordinate, we boost we result near this coordinate
        let boost_on_proximity =
            rs_q::build_function_score()
                .with_boost_mode(rs_es::query::compound::BoostMode::Multiply)
                .with_boost(500)
                .with_function(Function::build_decay("coord",
                                           rs_u::Location::LatLon(c.lat, c.lon),
                                           rs_u::Distance::new(50f64,
                                                               rs_u::DistanceUnit::Kilometer))
                                   .build_exp())
                .build();
        should_query.push(boost_on_proximity);
    } else {
        // if we don't have coords, we take the field `weight` into account
        let boost_on_weight = rs_q::build_function_score()
            .with_boost_mode(rs_es::query::compound::BoostMode::Multiply)
            .with_boost(300)
            .with_query(rs_q::build_match_all().build())
            .with_function(Function::build_field_value_factor("weight")
                .with_factor(1)
                .with_modifier(rs_es::query::functions::Modifier::Log1p)
                .build())
            .build();
        should_query.push(boost_on_weight);
    }

    let sub_query = rs_q::build_bool()
        .with_should(should_query)
        .build();

    use rs_es::query::{CombinationMinimumShouldMatch, MinimumShouldMatch};
    let min_match = |min, threshold| {
        CombinationMinimumShouldMatch::new(MinimumShouldMatch::from(min),
                                           MinimumShouldMatch::from(threshold))
    };

    let minimum_should_match = match match_type {
        MatchType::Prefix => {
            // when we search [0, 2] fields, we need to match 100% of them
            // when we search [3, 4] fields, we need to match 90% of them
            // when we search [5, +] fields, we only need to match 70% on them
            // this way if the input has more fields than the documents
            // (additional information like the country, the region, ...)
            // they can be ignored
            MinimumShouldMatch::from(vec![min_match(2, 90f64), min_match(4, 70f64)])
        }
        // for fuzzy search we lower our expectation and we accept 50% of token match
        MatchType::Fuzzy => MinimumShouldMatch::from(50f64),
    };

    let mut must = vec![rs_q::build_multi_match(vec![main_match_type.to_string(),
                                                     "zip_codes.prefix".to_string()],
                                                q.to_string())
                            .with_minimum_should_match(minimum_should_match)
                            .build()];

    if let Some(s) = shape {
        must.push(rs_q::build_geo_polygon("coord", s).build());
    }

    // filter to handle house number
    // we either want:
    // * to exactly match the document house_number
    // * or that the document has no house_number
    let filter = rs_q::build_bool()
        .with_should(vec![rs_q::build_bool()
                              .with_must_not(rs_q::build_exists("house_number").build())
                              .build(),
                          rs_q::build_match("house_number", q.to_string()).build()])
        .with_must(must)
        .build();

    rs_q::build_bool()
        .with_must(vec![sub_query])
        .with_filter(filter)
        .build()
}

fn is_existing_index(client: &mut rs_es::Client, index: &String) -> bool {
    match client.open_index(index) {
        Ok(_) => true,
        Err(_) => {
            debug!("{} index not found", index);
            false
        }
    }
}

fn make_indexes<'a>(pt_dataset_index: &'a Option<String>, client: &mut rs_es::Client) -> Vec<&'a str> {
    match *pt_dataset_index {
        Some(ref dataset) if ! dataset.is_empty() => {
            let mut result:Vec<&str> = vec![];
            if is_existing_index(client, &dataset) {
                result.push(dataset);
            }
            if is_existing_index(client, &"munin_street_network".to_string()) {
                result.push("munin_street_network");
            }
            result
        },
        _ => vec!["munin"]
    }

}

fn query(q: &str,
         pt_dataset: &Option<&str>,
         cnx: &str,
         match_type: MatchType,
         offset: u64,
         limit: u64,
         coord: &Option<model::Coord>,
         shape: Option<Vec<rs_es::units::Location>>)
         -> Result<Vec<mimir::Place>, rs_es::error::EsError> {
    let query = build_query(q, match_type, coord, shape);

    let mut client = build_rs_client(&cnx.to_string());

	let pt_dataset_index = pt_dataset.map(|d| format!("munin_stops_{}", d));
    let indexes = make_indexes(&pt_dataset_index, &mut client);

    let result: SearchResult<serde_json::Value> = try!(client.search_query()
        .with_indexes(&indexes)
        .with_query(&query)
        .with_from(offset)
        .with_size(limit)
        .send());

    debug!("{} documents found", result.hits.total);

    // for the moment rs-es does not handle enum Document,
    // so we need to convert the ES glob to a Place
    Ok(result.hits
        .hits
        .into_iter()
        .filter_map(|hit| make_place(hit.doc_type, hit.source))
        .collect())
}

pub fn autocomplete(q: &str,
                    pt_dataset: &Option<&str>,
                    offset: u64,
                    limit: u64,
                    coord: Option<model::Coord>,
                    cnx: &str,
                    shape: Option<Vec<(f64, f64)>>)
                    -> Result<Vec<mimir::Place>, rs_es::error::EsError> {
    fn make_shape(shape: &Option<Vec<(f64, f64)>>) -> Option<Vec<rs_es::units::Location>> {
        shape.as_ref().map(|v| v.iter().map(|&l| l.into()).collect())
    }

    // First we try a prety exact match on the prefix.
    // If there are no results then we do a new fuzzy search (matching ngrams)
    let results = try!(query(&q,
                             &pt_dataset,
                             cnx,
                             MatchType::Prefix,
                             offset,
                             limit,
                             &coord,
                             make_shape(&shape)));
    if results.is_empty() {
        query(&q,
              &pt_dataset,
              cnx,
              MatchType::Fuzzy,
              offset,
              limit,
              &coord,
              make_shape(&shape))
    } else {
        Ok(results)
    }
}
<commit_msg>fmt applied<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io
use super::model;
use regex;
use rs_es;
use rs_es::query::Query as rs_q;
use rs_es::operations::search::SearchResult;
use rs_es::units as rs_u;
use mimir;
use serde_json;
use serde;

fn build_rs_client(cnx: &String) -> rs_es::Client {
    let re = regex::Regex::new(r"(?:https?://)?(?P<host>.+?):(?P<port>\d+)").unwrap();
    let cap = re.captures(&cnx).unwrap();
    let host = cap.name("host").unwrap().as_str();
    let port = cap.name("port").unwrap().as_str().parse::<u32>().unwrap();

    rs_es::Client::new(&host, port)
}

/// takes a ES json blob and build a Place from it
/// it uses the _type field of ES to know which type of the Place enum to fill
pub fn make_place(doc_type: String, value: Option<Box<serde_json::Value>>) -> Option<mimir::Place> {
    value.and_then(|v| {
        fn convert<T: serde::Deserialize>(v: serde_json::Value,
                                          f: fn(T) -> mimir::Place)
                                          -> Option<mimir::Place> {
            serde_json::from_value::<T>(v)
                .map_err(|err| warn!("Impossible to load ES result: {}", err))
                .ok()
                .map(f)
        }
        match doc_type.as_ref() {
            "addr" => convert(*v, mimir::Place::Addr),
            "street" => convert(*v, mimir::Place::Street),
            "admin" => convert(*v, mimir::Place::Admin),
            "poi" => convert(*v, mimir::Place::Poi),
            "stop" => convert(*v, mimir::Place::Stop),
            _ => {
                warn!("unknown ES return value, _type field = {}", doc_type);
                None
            }
        }
    })
}

#[derive(Debug, Eq, PartialEq)]
enum MatchType {
    Prefix,
    Fuzzy,
}

fn build_query(q: &str,
               match_type: MatchType,
               coord: &Option<model::Coord>,
               shape: Option<Vec<rs_es::units::Location>>)
               -> rs_es::query::Query {
    use rs_es::query::functions::Function;
    // we order the type of object we want
    // Note: the addresses are boosted more because even if we don't want them first
    // because they are more severely filtered
    let boost_addr = rs_q::build_term("_type", "addr").with_boost(5000).build();
    let boost_admin = rs_q::build_term("_type", "admin").with_boost(3000).build();
    let boost_stop = rs_q::build_term("_type", "stop").with_boost(2000).build();

    let main_match_type = match match_type {
        MatchType::Prefix => "label.prefix",
        MatchType::Fuzzy => "label.ngram",
    };

    let boost_main_match_query = rs_q::build_multi_match(vec![main_match_type.to_string()],
                                                         q.to_string())
        .with_boost(100)
        .build();
    let boost_zipcode_match_query = rs_q::build_multi_match(vec!["zip_codes.prefix".to_string()],
                                                            q.to_string())
        .with_boost(500)
        .build();

    let mut should_query = vec![boost_addr,
                                boost_admin,
                                boost_stop,
                                boost_main_match_query,
                                boost_zipcode_match_query];

    // for fuzzy search we also search by the prefix index (with a greater boost than ngram)
    // to have better results
    if match_type == MatchType::Fuzzy {
        should_query.push(rs_q::build_match("label.prefix", q.to_string())
            .with_boost(1000)
            .build());
    }

    if let &Some(ref c) = coord {
        // if we have coordinate, we boost we result near this coordinate
        let boost_on_proximity =
            rs_q::build_function_score()
                .with_boost_mode(rs_es::query::compound::BoostMode::Multiply)
                .with_boost(500)
                .with_function(Function::build_decay("coord",
                                           rs_u::Location::LatLon(c.lat, c.lon),
                                           rs_u::Distance::new(50f64,
                                                               rs_u::DistanceUnit::Kilometer))
                                   .build_exp())
                .build();
        should_query.push(boost_on_proximity);
    } else {
        // if we don't have coords, we take the field `weight` into account
        let boost_on_weight = rs_q::build_function_score()
            .with_boost_mode(rs_es::query::compound::BoostMode::Multiply)
            .with_boost(300)
            .with_query(rs_q::build_match_all().build())
            .with_function(Function::build_field_value_factor("weight")
                .with_factor(1)
                .with_modifier(rs_es::query::functions::Modifier::Log1p)
                .build())
            .build();
        should_query.push(boost_on_weight);
    }

    let sub_query = rs_q::build_bool()
        .with_should(should_query)
        .build();

    use rs_es::query::{CombinationMinimumShouldMatch, MinimumShouldMatch};
    let min_match = |min, threshold| {
        CombinationMinimumShouldMatch::new(MinimumShouldMatch::from(min),
                                           MinimumShouldMatch::from(threshold))
    };

    let minimum_should_match = match match_type {
        MatchType::Prefix => {
            // when we search [0, 2] fields, we need to match 100% of them
            // when we search [3, 4] fields, we need to match 90% of them
            // when we search [5, +] fields, we only need to match 70% on them
            // this way if the input has more fields than the documents
            // (additional information like the country, the region, ...)
            // they can be ignored
            MinimumShouldMatch::from(vec![min_match(2, 90f64), min_match(4, 70f64)])
        }
        // for fuzzy search we lower our expectation and we accept 50% of token match
        MatchType::Fuzzy => MinimumShouldMatch::from(50f64),
    };

    let mut must = vec![rs_q::build_multi_match(vec![main_match_type.to_string(),
                                                     "zip_codes.prefix".to_string()],
                                                q.to_string())
                            .with_minimum_should_match(minimum_should_match)
                            .build()];

    if let Some(s) = shape {
        must.push(rs_q::build_geo_polygon("coord", s).build());
    }

    // filter to handle house number
    // we either want:
    // * to exactly match the document house_number
    // * or that the document has no house_number
    let filter = rs_q::build_bool()
        .with_should(vec![rs_q::build_bool()
                              .with_must_not(rs_q::build_exists("house_number").build())
                              .build(),
                          rs_q::build_match("house_number", q.to_string()).build()])
        .with_must(must)
        .build();

    rs_q::build_bool()
        .with_must(vec![sub_query])
        .with_filter(filter)
        .build()
}

fn is_existing_index(client: &mut rs_es::Client, index: &String) -> bool {
    match client.open_index(index) {
        Ok(_) => true,
        Err(_) => {
            debug!("{} index not found", index);
            false
        }
    }
}

fn make_indexes<'a>(pt_dataset_index: &'a Option<String>,
                    client: &mut rs_es::Client)
                    -> Vec<&'a str> {
    match *pt_dataset_index {
        Some(ref dataset) if !dataset.is_empty() => {
            let mut result: Vec<&str> = vec![];
            if is_existing_index(client, &dataset) {
                result.push(dataset);
            }
            if is_existing_index(client, &"munin_street_network".to_string()) {
                result.push("munin_street_network");
            }
            result
        }
        _ => vec!["munin"],
    }

}

fn query(q: &str,
         pt_dataset: &Option<&str>,
         cnx: &str,
         match_type: MatchType,
         offset: u64,
         limit: u64,
         coord: &Option<model::Coord>,
         shape: Option<Vec<rs_es::units::Location>>)
         -> Result<Vec<mimir::Place>, rs_es::error::EsError> {
    let query = build_query(q, match_type, coord, shape);

    let mut client = build_rs_client(&cnx.to_string());

    let pt_dataset_index = pt_dataset.map(|d| format!("munin_stops_{}", d));
    let indexes = make_indexes(&pt_dataset_index, &mut client);

    let result: SearchResult<serde_json::Value> = try!(client.search_query()
        .with_indexes(&indexes)
        .with_query(&query)
        .with_from(offset)
        .with_size(limit)
        .send());

    debug!("{} documents found", result.hits.total);

    // for the moment rs-es does not handle enum Document,
    // so we need to convert the ES glob to a Place
    Ok(result.hits
        .hits
        .into_iter()
        .filter_map(|hit| make_place(hit.doc_type, hit.source))
        .collect())
}

pub fn autocomplete(q: &str,
                    pt_dataset: &Option<&str>,
                    offset: u64,
                    limit: u64,
                    coord: Option<model::Coord>,
                    cnx: &str,
                    shape: Option<Vec<(f64, f64)>>)
                    -> Result<Vec<mimir::Place>, rs_es::error::EsError> {
    fn make_shape(shape: &Option<Vec<(f64, f64)>>) -> Option<Vec<rs_es::units::Location>> {
        shape.as_ref().map(|v| v.iter().map(|&l| l.into()).collect())
    }

    // First we try a prety exact match on the prefix.
    // If there are no results then we do a new fuzzy search (matching ngrams)
    let results = try!(query(&q,
                             &pt_dataset,
                             cnx,
                             MatchType::Prefix,
                             offset,
                             limit,
                             &coord,
                             make_shape(&shape)));
    if results.is_empty() {
        query(&q,
              &pt_dataset,
              cnx,
              MatchType::Fuzzy,
              offset,
              limit,
              &coord,
              make_shape(&shape))
    } else {
        Ok(results)
    }
}
<|endoftext|>


<commit_before>//!
//! The API for communicating with devices.
//!
//! This API is provided as Traits to be implemented:
//!
//! - by the low-level layers of the FoxBox, including the adapters;
//! - by test suites and tools that need to simulate connected devices.
//!
//! In turn, this API is used to implement:
//!
//! - the public-facing REST and WebSocket API;
//! - the rules API (ThinkerBell).
//!
//!

use devices::*;
use requests::*;
use values::Value;

/// An error produced by one of the APIs in this module.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub enum Error {
    /// There is no such node connected to the Foxbox, even indirectly.
    NoSuchNode(NodeId),

    /// There is no such service connected to the Foxbox, even indirectly.
    NoSuchService(ServiceId),

    /// Attempting to set a value with the wrong type
    TypeError,
}

/// An event during watching.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub enum WatchEvent {
    /// A new value is available.
    Value {
        /// The service that sent the value.
        from: ServiceId,

        /// The actual value.
        value: Value
    },

    /// The set of devices being watched has changed, typically either
    /// because a tag was edited or because a device was
    /// removed. Payload is the id of the device that was removed.
    InputRemoved(ServiceId),

    /// The set of devices being watched has changed, typically either
    /// because a tag was edited or because a device was
    /// added. Payload is the id of the device that was added.
    InputAdded(ServiceId),
}

/// The public API.
pub trait API {
    /// Get the metadata on nodes matching some conditions.
    ///
    /// A call to `API::get_nodes(vec![req1, req2, ...])` will return
    /// the metadata on all nodes matching _either_ `req1` or `req2`
    /// or ...
    ///
    /// # REST API
    ///
    /// `GET /api/v1/nodes`
    ///
    /// ## Inputs
    ///
    /// Any JSON that can be deserialized to a `Vec<NodeRequest>`. See
    /// the implementation of `NodeRequest` for details.
    ///
    /// ### Example
    ///
    /// Request all doors in the entrance (tags `door`, `entrance`)
    /// that support output service `OpenClose`
    ///
    /// ```json
    /// [{
    ///   "tags": ["entrance", "door"],
    ///   "inputs": [
    ///     {
    ///       "kind": {
    ///         "Exactly": {
    ///           "OpenClose": []
    ///         }
    ///       }
    ///     }
    ///   ]
    /// }]
    /// ```
    ///
    ///
    /// ## Errors
    ///
    /// In case of syntax error, Error 400, accompanied with a
    /// somewhat human-readable JSON string detailing the error.
    ///
    /// ## Success
    ///
    /// A JSON representing an array of `Node`. See the implementation
    /// of `Node` for details.
    ///
    /// ### Example
    ///
    /// ```json
    /// [{
    ///   "tags": ["entrance", "door", "somevendor"],
    ///   "id: "some-node-id",
    ///   "inputs": [],
    ///   "outputs": [
    ///     "tags": [...],
    ///     "id": "some-service-id",
    ///     "node": "some-node-id",
    ///     "last_seen": "some-date",
    ///     "mechanism": {
    ///       "Output":  {
    ///         "kind": {
    ///           "OnOff": []
    ///         },
    ///         "push": [5000],
    ///         "updated": "some-date",
    ///       }
    ///     }
    ///   ]
    /// }]
    /// ```
    fn get_nodes(&Vec<NodeRequest>) -> Vec<Node>;

    /// Label a set of nodes with a set of tags.
    ///
    /// A call to `API::put_node_tag(vec![req1, req2, ...], vec![tag1,
    /// ...])` will label all the nodes matching _either_ `req1` or
    /// `req2` or ... with `tag1`, ... and return the number of nodes
    /// matching any of the requests.
    ///
    /// Some of the nodes may already be labelled with `tag1`, or
    /// `tag2`, ... They will not change state. They are counted in
    /// the resulting `usize` nevertheless.
    ///
    /// Note that this call is _not live_. In other words, if nodes
    /// are added after the call, they will not be affected.
    ///
    /// # REST API
    ///
    /// `POST /api/v1/nodes/tag`
    ///
    /// ## Inputs
    ///
    /// Any JSON that can be deserialized to
    ///
    /// ```ignore
    /// {
    ///   set: Vec<NodeRequest>,
    ///   tags: Vec<String>,
    /// }
    /// ```
    ///
    /// ## Errors
    ///
    /// In case of syntax error, Error 400, accompanied with a
    /// somewhat human-readable JSON string detailing the error.
    ///
    /// ## Success
    ///
    /// A JSON string representing a number.
    fn put_node_tag(set: &Vec<NodeRequest>, tags: &Vec<String>) -> usize;

    /// Remove a set of tags from a set of nodes.
    ///
    /// A call to `API::delete_node_tag(vec![req1, req2, ...], vec![tag1,
    /// ...])` will remove from all the nodes matching _either_ `req1` or
    /// `req2` or ... all of the tags `tag1`, ... and return the number of nodes
    /// matching any of the requests.
    ///
    /// Some of the nodes may not be labelled with `tag1`, or `tag2`,
    /// ... They will not change state. They are counted in the
    /// resulting `usize` nevertheless.
    ///
    /// Note that this call is _not live_. In other words, if nodes
    /// are added after the call, they will not be affected.
    ///
    /// # REST API
    ///
    /// `DELETE /api/v1/nodes/tag`
    ///
    /// ## Inputs
    ///
    /// Any JSON that can be deserialized to
    ///
    /// ```ignore
    /// {
    ///   set: Vec<NodeRequest>,
    ///   tags: Vec<String>,
    /// }
    /// ```
    ///
    /// ## Errors
    ///
    /// In case of syntax error, Error 400, accompanied with a
    /// somewhat human-readable JSON string detailing the error.
    ///
    /// ## Success
    ///
    /// A JSON representing a number.
    fn delete_node_tag(set: &Vec<NodeRequest>, tags: String) -> usize;

    /// Get a list of inputs matching some conditions
    ///
    /// # REST API
    ///
    /// `GET /api/v1/services`
    fn get_input_services(&Vec<InputRequest>) -> Vec<Service<Input>>;
    fn get_output_services(&Vec<OutputRequest>) -> Vec<Service<Output>>;

    /// Label a set of services with a set of tags.
    ///
    /// A call to `API::put_{input, output}_tag(vec![req1, req2, ...], vec![tag1,
    /// ...])` will label all the services matching _either_ `req1` or
    /// `req2` or ... with `tag1`, ... and return the number of services
    /// matching any of the requests.
    ///
    /// Some of the services may already be labelled with `tag1`, or
    /// `tag2`, ... They will not change state. They are counted in
    /// the resulting `usize` nevertheless.
    ///
    /// Note that this call is _not live_. In other words, if services
    /// are added after the call, they will not be affected.
    ///
    /// # REST API
    ///
    /// `POST /api/v1/services/tag`
    ///
    /// ## Inputs
    ///
    /// Any JSON that can be deserialized to
    ///
    /// ```ignore
    /// {
    ///   set: Vec<InputRequest>,
    ///   tags: Vec<String>,
    /// }
    /// ```
    /// or
    /// ```ignore
    /// {
    ///   set: Vec<OutputRequest>,
    ///   tags: Vec<String>,
    /// }
    /// ```
    ///
    /// ## Errors
    ///
    /// In case of syntax error, Error 400, accompanied with a
    /// somewhat human-readable JSON string detailing the error.
    ///
    /// ## Success
    ///
    /// A JSON representing a number.
    fn put_input_tag(&Vec<InputRequest>, &Vec<String>) -> usize;
    fn put_output_tag(&Vec<OutputRequest>, &Vec<String>) -> usize;

    /// Remove a set of tags from a set of services.
    ///
    /// A call to `API::delete_{input, output}_tag(vec![req1, req2, ...], vec![tag1,
    /// ...])` will remove from all the services matching _either_ `req1` or
    /// `req2` or ... all of the tags `tag1`, ... and return the number of services
    /// matching any of the requests.
    ///
    /// Some of the services may not be labelled with `tag1`, or `tag2`,
    /// ... They will not change state. They are counted in the
    /// resulting `usize` nevertheless.
    ///
    /// Note that this call is _not live_. In other words, if services
    /// are added after the call, they will not be affected.
    ///
    /// # REST API
    ///
    /// `DELETE /api/v1/services/tag`
    ///
    /// ## Inputs
    ///
    /// Any JSON that can be deserialized to
    ///
    /// ```ignore
    /// {
    ///   set: Vec<InputRequest>,
    ///   tags: Vec<String>,
    /// }
    /// ```
    /// or
    /// ```ignore
    /// {
    ///   set: Vec<OutputRequest>,
    ///   tags: Vec<String>,
    /// }
    /// ```
    ///
    /// ## Errors
    ///
    /// In case of syntax error, Error 400, accompanied with a
    /// somewhat human-readable JSON string detailing the error.
    ///
    /// ## Success
    ///
    /// A JSON representing a number.
    fn delete_input_tag(&Vec<InputRequest>, &Vec<String>) -> usize;
    fn delete_output_tag(&Vec<InputRequest>, &Vec<String>) -> usize;

    /// Read the latest value from a set of services
    ///
    /// # REST API
    ///
    /// `GET /api/v1/services/value`
    fn get_service_value(&Vec<InputRequest>) -> Vec<(ServiceId, Result<Value, Error>)>;

    /// Send one value to a set of services
    ///
    /// # REST API
    ///
    /// `POST /api/v1/services/value`
    fn put_service_value(&Vec<OutputRequest>, Value) -> Vec<(ServiceId, Result<(), Error>)>;

    /// Watch for any change
    ///
    /// # WebSocket API
    ///
    /// `/api/v1/services/watch`
    fn register_service_watch<F>(Vec<WatchOptions>, cb: F) -> Self::WatchGuard
        where F: FnMut(WatchEvent) + Send + 'static;

    /// A value that causes a disconnection once it is dropped.
    type WatchGuard;
}

/// Options for watching changes in one or more services.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct WatchOptions {
    /// The set of inputs to watch. Note that the actual inputs in the
    /// set may change over time.
    source: InputRequest,

    /// If `true`, watch as new values become available.
    should_watch_values: bool,

    /// If `true`, watch as nodes are connected/disconnected.
    should_watch_topology: bool,
}

impl WatchOptions {
    pub fn new() -> Self {
        WatchOptions {
            source: InputRequest::new(),
            should_watch_values: false,
            should_watch_topology: false,
        }
    }

    /// Restrict to input services in a given set.
    ///
    /// Also note that the actual input services that are part of the
    /// set may change with time, for instance if devices are added
    /// ore removed.  The request _is live_, i.e. the service watch
    /// will continue watching any input services that match `req`.
    pub fn with_inputs(self, req: InputRequest) -> Self {
        WatchOptions {
            source: self.source.and(req),
            ..self
        }
    }

    pub fn with_watch_values(self, should: bool) -> Self {
        WatchOptions {
            should_watch_values: should,
            ..self
        }
    }

    pub fn with_watch_topology(self, should: bool) -> Self {
        WatchOptions {
            should_watch_topology: should,
            ..self
        }
    }
}
<commit_msg>API is now once again designed to be a Trait object<commit_after>//!
//! The API for communicating with devices.
//!
//! This API is provided as Traits to be implemented:
//!
//! - by the low-level layers of the FoxBox, including the adapters;
//! - by test suites and tools that need to simulate connected devices.
//!
//! In turn, this API is used to implement:
//!
//! - the public-facing REST and WebSocket API;
//! - the rules API (ThinkerBell).
//!
//!

use devices::*;
use requests::*;
use values::Value;

/// An error produced by one of the APIs in this module.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub enum Error {
    /// There is no such node connected to the Foxbox, even indirectly.
    NoSuchNode(NodeId),

    /// There is no such service connected to the Foxbox, even indirectly.
    NoSuchService(ServiceId),

    /// Attempting to set a value with the wrong type
    TypeError,
}

/// An event during watching.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub enum WatchEvent {
    /// A new value is available.
    Value {
        /// The service that sent the value.
        from: ServiceId,

        /// The actual value.
        value: Value
    },

    /// The set of devices being watched has changed, typically either
    /// because a tag was edited or because a device was
    /// removed. Payload is the id of the device that was removed.
    InputRemoved(ServiceId),

    /// The set of devices being watched has changed, typically either
    /// because a tag was edited or because a device was
    /// added. Payload is the id of the device that was added.
    InputAdded(ServiceId),
}

/// The public API.
pub trait API {
    /// Get the metadata on nodes matching some conditions.
    ///
    /// A call to `API::get_nodes(vec![req1, req2, ...])` will return
    /// the metadata on all nodes matching _either_ `req1` or `req2`
    /// or ...
    ///
    /// # REST API
    ///
    /// `GET /api/v1/nodes`
    ///
    /// ## Inputs
    ///
    /// Any JSON that can be deserialized to a `Vec<NodeRequest>`. See
    /// the implementation of `NodeRequest` for details.
    ///
    /// ### Example
    ///
    /// Request all doors in the entrance (tags `door`, `entrance`)
    /// that support output service `OpenClose`
    ///
    /// ```json
    /// [{
    ///   "tags": ["entrance", "door"],
    ///   "inputs": [
    ///     {
    ///       "kind": {
    ///         "Exactly": {
    ///           "OpenClose": []
    ///         }
    ///       }
    ///     }
    ///   ]
    /// }]
    /// ```
    ///
    ///
    /// ## Errors
    ///
    /// In case of syntax error, Error 400, accompanied with a
    /// somewhat human-readable JSON string detailing the error.
    ///
    /// ## Success
    ///
    /// A JSON representing an array of `Node`. See the implementation
    /// of `Node` for details.
    ///
    /// ### Example
    ///
    /// ```json
    /// [{
    ///   "tags": ["entrance", "door", "somevendor"],
    ///   "id: "some-node-id",
    ///   "inputs": [],
    ///   "outputs": [
    ///     "tags": [...],
    ///     "id": "some-service-id",
    ///     "node": "some-node-id",
    ///     "last_seen": "some-date",
    ///     "mechanism": {
    ///       "Output":  {
    ///         "kind": {
    ///           "OnOff": []
    ///         },
    ///         "push": [5000],
    ///         "updated": "some-date",
    ///       }
    ///     }
    ///   ]
    /// }]
    /// ```
    fn get_nodes(&self, &Vec<NodeRequest>) -> Vec<Node>;

    /// Label a set of nodes with a set of tags.
    ///
    /// A call to `API::put_node_tag(vec![req1, req2, ...], vec![tag1,
    /// ...])` will label all the nodes matching _either_ `req1` or
    /// `req2` or ... with `tag1`, ... and return the number of nodes
    /// matching any of the requests.
    ///
    /// Some of the nodes may already be labelled with `tag1`, or
    /// `tag2`, ... They will not change state. They are counted in
    /// the resulting `usize` nevertheless.
    ///
    /// Note that this call is _not live_. In other words, if nodes
    /// are added after the call, they will not be affected.
    ///
    /// # REST API
    ///
    /// `POST /api/v1/nodes/tag`
    ///
    /// ## Inputs
    ///
    /// Any JSON that can be deserialized to
    ///
    /// ```ignore
    /// {
    ///   set: Vec<NodeRequest>,
    ///   tags: Vec<String>,
    /// }
    /// ```
    ///
    /// ## Errors
    ///
    /// In case of syntax error, Error 400, accompanied with a
    /// somewhat human-readable JSON string detailing the error.
    ///
    /// ## Success
    ///
    /// A JSON string representing a number.
    fn put_node_tag(&self, set: &Vec<NodeRequest>, tags: &Vec<String>) -> usize;

    /// Remove a set of tags from a set of nodes.
    ///
    /// A call to `API::delete_node_tag(vec![req1, req2, ...], vec![tag1,
    /// ...])` will remove from all the nodes matching _either_ `req1` or
    /// `req2` or ... all of the tags `tag1`, ... and return the number of nodes
    /// matching any of the requests.
    ///
    /// Some of the nodes may not be labelled with `tag1`, or `tag2`,
    /// ... They will not change state. They are counted in the
    /// resulting `usize` nevertheless.
    ///
    /// Note that this call is _not live_. In other words, if nodes
    /// are added after the call, they will not be affected.
    ///
    /// # REST API
    ///
    /// `DELETE /api/v1/nodes/tag`
    ///
    /// ## Inputs
    ///
    /// Any JSON that can be deserialized to
    ///
    /// ```ignore
    /// {
    ///   set: Vec<NodeRequest>,
    ///   tags: Vec<String>,
    /// }
    /// ```
    ///
    /// ## Errors
    ///
    /// In case of syntax error, Error 400, accompanied with a
    /// somewhat human-readable JSON string detailing the error.
    ///
    /// ## Success
    ///
    /// A JSON representing a number.
    fn delete_node_tag(&self, set: &Vec<NodeRequest>, tags: String) -> usize;

    /// Get a list of inputs matching some conditions
    ///
    /// # REST API
    ///
    /// `GET /api/v1/services`
    fn get_input_services(&self, &Vec<InputRequest>) -> Vec<Service<Input>>;
    fn get_output_services(&self, &Vec<OutputRequest>) -> Vec<Service<Output>>;

    /// Label a set of services with a set of tags.
    ///
    /// A call to `API::put_{input, output}_tag(vec![req1, req2, ...], vec![tag1,
    /// ...])` will label all the services matching _either_ `req1` or
    /// `req2` or ... with `tag1`, ... and return the number of services
    /// matching any of the requests.
    ///
    /// Some of the services may already be labelled with `tag1`, or
    /// `tag2`, ... They will not change state. They are counted in
    /// the resulting `usize` nevertheless.
    ///
    /// Note that this call is _not live_. In other words, if services
    /// are added after the call, they will not be affected.
    ///
    /// # REST API
    ///
    /// `POST /api/v1/services/tag`
    ///
    /// ## Inputs
    ///
    /// Any JSON that can be deserialized to
    ///
    /// ```ignore
    /// {
    ///   set: Vec<InputRequest>,
    ///   tags: Vec<String>,
    /// }
    /// ```
    /// or
    /// ```ignore
    /// {
    ///   set: Vec<OutputRequest>,
    ///   tags: Vec<String>,
    /// }
    /// ```
    ///
    /// ## Errors
    ///
    /// In case of syntax error, Error 400, accompanied with a
    /// somewhat human-readable JSON string detailing the error.
    ///
    /// ## Success
    ///
    /// A JSON representing a number.
    fn put_input_tag(&self, &Vec<InputRequest>, &Vec<String>) -> usize;
    fn put_output_tag(&self, &Vec<OutputRequest>, &Vec<String>) -> usize;

    /// Remove a set of tags from a set of services.
    ///
    /// A call to `API::delete_{input, output}_tag(vec![req1, req2, ...], vec![tag1,
    /// ...])` will remove from all the services matching _either_ `req1` or
    /// `req2` or ... all of the tags `tag1`, ... and return the number of services
    /// matching any of the requests.
    ///
    /// Some of the services may not be labelled with `tag1`, or `tag2`,
    /// ... They will not change state. They are counted in the
    /// resulting `usize` nevertheless.
    ///
    /// Note that this call is _not live_. In other words, if services
    /// are added after the call, they will not be affected.
    ///
    /// # REST API
    ///
    /// `DELETE /api/v1/services/tag`
    ///
    /// ## Inputs
    ///
    /// Any JSON that can be deserialized to
    ///
    /// ```ignore
    /// {
    ///   set: Vec<InputRequest>,
    ///   tags: Vec<String>,
    /// }
    /// ```
    /// or
    /// ```ignore
    /// {
    ///   set: Vec<OutputRequest>,
    ///   tags: Vec<String>,
    /// }
    /// ```
    ///
    /// ## Errors
    ///
    /// In case of syntax error, Error 400, accompanied with a
    /// somewhat human-readable JSON string detailing the error.
    ///
    /// ## Success
    ///
    /// A JSON representing a number.
    fn delete_input_tag(&self, &Vec<InputRequest>, &Vec<String>) -> usize;
    fn delete_output_tag(&self, &Vec<InputRequest>, &Vec<String>) -> usize;

    /// Read the latest value from a set of services
    ///
    /// # REST API
    ///
    /// `GET /api/v1/services/value`
    fn get_service_value(&self, &Vec<InputRequest>) -> Vec<(ServiceId, Result<Value, Error>)>;

    /// Send one value to a set of services
    ///
    /// # REST API
    ///
    /// `POST /api/v1/services/value`
    fn put_service_value(&self, &Vec<OutputRequest>, Value) -> Vec<(ServiceId, Result<(), Error>)>;

    /// Watch for any change
    ///
    /// # WebSocket API
    ///
    /// `/api/v1/services/watch`
    fn register_service_watch(&self, Vec<WatchOptions>, cb: Box<FnMut(WatchEvent) + Send + 'static>) -> Self::WatchGuard;

    /// A value that causes a disconnection once it is dropped.
    type WatchGuard;
}

/// Options for watching changes in one or more services.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct WatchOptions {
    /// The set of inputs to watch. Note that the actual inputs in the
    /// set may change over time.
    source: InputRequest,

    /// If `true`, watch as new values become available.
    should_watch_values: bool,

    /// If `true`, watch as nodes are connected/disconnected.
    should_watch_topology: bool,
}

impl WatchOptions {
    pub fn new() -> Self {
        WatchOptions {
            source: InputRequest::new(),
            should_watch_values: false,
            should_watch_topology: false,
        }
    }

    /// Restrict to input services in a given set.
    ///
    /// Also note that the actual input services that are part of the
    /// set may change with time, for instance if devices are added
    /// ore removed.  The request _is live_, i.e. the service watch
    /// will continue watching any input services that match `req`.
    pub fn with_inputs(self, req: InputRequest) -> Self {
        WatchOptions {
            source: self.source.and(req),
            ..self
        }
    }

    pub fn with_watch_values(self, should: bool) -> Self {
        WatchOptions {
            should_watch_values: should,
            ..self
        }
    }

    pub fn with_watch_topology(self, should: bool) -> Self {
        WatchOptions {
            should_watch_topology: should,
            ..self
        }
    }
}
<|endoftext|>