<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

//! A task that takes a URL and streams back the binary data.

use about_loader;
use data_loader;
use file_loader;
use http_loader;
use cookie_storage::CookieStorage;
use cookie;
use mime_classifier::MIMEClassifier;

use net_traits::{ControlMsg, LoadData, LoadResponse, LoadConsumer};
use net_traits::{Metadata, ProgressMsg, ResourceTask, AsyncResponseTarget, ResponseAction};
use net_traits::ProgressMsg::Done;
use util::opts;
use util::task::spawn_named;

use devtools_traits::{DevtoolsControlMsg};
use hyper::header::{ContentType, Header, SetCookie, UserAgent};
use hyper::mime::{Mime, TopLevel, SubLevel};

use std::borrow::ToOwned;
use std::boxed::{self, FnBox};
use std::collections::HashMap;
use std::env;
use std::fs::File;
use std::io::{BufReader, Read};
use std::str::FromStr;
use std::sync::Arc;
use std::sync::mpsc::{channel, Receiver, Sender};

static mut HOST_TABLE: Option<*mut HashMap<String, String>> = None;

pub fn global_init() {
    //TODO: handle bad file path
    let path = match env::var("HOST_FILE") {
        Ok(host_file_path) => host_file_path,
        Err(_) => return,
    };

    let mut file = match File::open(&path) {
        Ok(f) => BufReader::new(f),
        Err(_) => return,
    };

    let mut lines = String::new();
    match file.read_to_string(&mut lines) {
        Ok(_) => (),
        Err(_) => return,
    };

    unsafe {
        let host_table = boxed::into_raw(parse_hostsfile(&lines));
        HOST_TABLE = Some(host_table);
    }
}

pub enum ProgressSender {
    Channel(Sender<ProgressMsg>),
    Listener(Box<AsyncResponseTarget>),
}

impl ProgressSender {
    //XXXjdm return actual error
    pub fn send(&self, msg: ProgressMsg) -> Result<(), ()> {
        match *self {
            ProgressSender::Channel(ref c) => c.send(msg).map_err(|_| ()),
            ProgressSender::Listener(ref b) => {
                let action = match msg {
                    ProgressMsg::Payload(buf) => ResponseAction::DataAvailable(buf),
                    ProgressMsg::Done(status) => ResponseAction::ResponseComplete(status),
                };
                b.invoke_with_listener(action);
                Ok(())
            }
        }
    }
}

/// For use by loaders in responding to a Load message.
pub fn start_sending(start_chan: LoadConsumer, metadata: Metadata) -> ProgressSender {
    start_sending_opt(start_chan, metadata).ok().unwrap()
}

/// For use by loaders in responding to a Load message that allows content sniffing.
pub fn start_sending_sniffed(start_chan: LoadConsumer, metadata: Metadata,
                             classifier: Arc<MIMEClassifier>, partial_body: &Vec<u8>)
                             -> ProgressSender {
    start_sending_sniffed_opt(start_chan, metadata, classifier, partial_body).ok().unwrap()
}

/// For use by loaders in responding to a Load message that allows content sniffing.
pub fn start_sending_sniffed_opt(start_chan: LoadConsumer, mut metadata: Metadata,
                                 classifier: Arc<MIMEClassifier>, partial_body: &Vec<u8>)
                                 -> Result<ProgressSender, ()> {
    if opts::get().sniff_mime_types {
        // TODO: should be calculated in the resource loader, from pull requeset #4094
        let nosniff = false;
        let check_for_apache_bug = false;

        let supplied_type = metadata.content_type.map(|ContentType(Mime(toplevel, sublevel, _))| {
            (format!("{}", toplevel), format!("{}", sublevel))
        });
        metadata.content_type = classifier.classify(nosniff, check_for_apache_bug, &supplied_type, &partial_body).map(|(toplevel, sublevel)| {
            let mime_tp: TopLevel = FromStr::from_str(&toplevel).unwrap();
            let mime_sb: SubLevel = FromStr::from_str(&sublevel).unwrap();
            ContentType(Mime(mime_tp, mime_sb, vec!()))
        });

    }

    start_sending_opt(start_chan, metadata)
}

/// For use by loaders in responding to a Load message.
pub fn start_sending_opt(start_chan: LoadConsumer, metadata: Metadata) -> Result<ProgressSender, ()> {
    match start_chan {
        LoadConsumer::Channel(start_chan) => {
            let (progress_chan, progress_port) = channel();
            let result = start_chan.send(LoadResponse {
                metadata:      metadata,
                progress_port: progress_port,
            });
            match result {
                Ok(_) => Ok(ProgressSender::Channel(progress_chan)),
                Err(_) => Err(())
            }
        }
        LoadConsumer::Listener(target) => {
            target.invoke_with_listener(ResponseAction::HeadersAvailable(metadata));
            Ok(ProgressSender::Listener(target))
        }
    }
}

/// Create a ResourceTask
pub fn new_resource_task(user_agent: Option<String>, devtools_chan: Option<Sender<DevtoolsControlMsg>>) -> ResourceTask {
    let (setup_chan, setup_port) = channel();
    let setup_chan_clone = setup_chan.clone();
    spawn_named("ResourceManager".to_owned(), move || {
        ResourceManager::new(setup_port, user_agent, setup_chan_clone, devtools_chan).start();
    });
    setup_chan
}

pub fn parse_hostsfile(hostsfile_content: &str) -> Box<HashMap<String, String>> {
    let ipv4_regex = regex!(r"^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$");
    let ipv6_regex = regex!(r"^([a-fA-F0-9]{0,4}[:]?){1,8}(/\d{1,3})?$");
    let mut host_table = HashMap::new();
    let lines: Vec<&str> = hostsfile_content.split('\n').collect();

    for line in lines.iter() {
        let ip_host: Vec<&str> = line.trim().split(|c: char| c == ' ' || c == '\t').collect();
        if ip_host.len() > 1 {
            if !ipv4_regex.is_match(ip_host[0]) && !ipv6_regex.is_match(ip_host[0]) { continue; }
            let address = ip_host[0].to_owned();

            for token in ip_host.iter().skip(1) {
                if token.as_bytes()[0] == b'#' {
                    break;
                }
                host_table.insert(token.to_owned().to_string(), address.clone());
            }
        }
    }
    box host_table
}

pub fn replace_hosts(mut load_data: LoadData, host_table: *mut HashMap<String, String>) -> LoadData {
    if let Some(h) = load_data.url.domain_mut() {
        unsafe {
            if let Some(ip) = (*host_table).get(h) {
                *h = ip.clone();
            }
        }
    }
    return load_data;
}

struct ResourceManager {
    from_client: Receiver<ControlMsg>,
    user_agent: Option<String>,
    cookie_storage: CookieStorage,
    resource_task: Sender<ControlMsg>,
    mime_classifier: Arc<MIMEClassifier>,
    devtools_chan: Option<Sender<DevtoolsControlMsg>>
}

impl ResourceManager {
    fn new(from_client: Receiver<ControlMsg>, user_agent: Option<String>,
           resource_task: Sender<ControlMsg>, devtools_channel: Option<Sender<DevtoolsControlMsg>>) -> ResourceManager {
        ResourceManager {
            from_client: from_client,
            user_agent: user_agent,
            cookie_storage: CookieStorage::new(),
            resource_task: resource_task,
            mime_classifier: Arc::new(MIMEClassifier::new()),
            devtools_chan: devtools_channel
        }
    }
}

impl ResourceManager {
    fn start(&mut self) {
        loop {
            match self.from_client.recv().unwrap() {
              ControlMsg::Load(load_data, consumer) => {
                self.load(load_data, consumer)
              }
              ControlMsg::SetCookiesForUrl(request, cookie_list, source) => {
                let header = Header::parse_header(&[cookie_list.into_bytes()]);
                if let Some(SetCookie(cookies)) = header {
                  for bare_cookie in cookies.into_iter() {
                    if let Some(cookie) = cookie::Cookie::new_wrapped(bare_cookie, &request, source) {
                      self.cookie_storage.push(cookie, source);
                    }
                  }
                }
              }
              ControlMsg::GetCookiesForUrl(url, consumer, source) => {
                consumer.send(self.cookie_storage.cookies_for_url(&url, source)).unwrap();
              }
              ControlMsg::Exit => {
                break
              }
            }
        }
    }

    fn load(&mut self, mut load_data: LoadData, consumer: LoadConsumer) {
        unsafe {
            if let Some(host_table) = HOST_TABLE {
                load_data = replace_hosts(load_data, host_table);
            }
        }

        self.user_agent.as_ref().map(|ua| load_data.headers.set(UserAgent(ua.clone())));

        fn from_factory(factory: fn(LoadData, LoadConsumer, Arc<MIMEClassifier>))
                        -> Box<FnBox(LoadData, LoadConsumer, Arc<MIMEClassifier>) + Send> {
            box move |load_data, senders, classifier| {
                factory(load_data, senders, classifier)
            }
        }

        let loader = match &*load_data.url.scheme {
            "file" => from_factory(file_loader::factory),
            "http" | "https" | "view-source" => http_loader::factory(self.resource_task.clone(), self.devtools_chan.clone()),
            "data" => from_factory(data_loader::factory),
            "about" => from_factory(about_loader::factory),
            _ => {
                debug!("resource_task: no loader for scheme {}", load_data.url.scheme);
                start_sending(consumer, Metadata::default(load_data.url))
                    .send(ProgressMsg::Done(Err("no loader for scheme".to_string()))).unwrap();
                return
            }
        };
        debug!("resource_task: loading url: {}", load_data.url.serialize());

        loader.call_box((load_data, consumer, self.mime_classifier.clone()));
    }
}
<commit_msg>fixes 4184: no-sniff and check-for-apache-bug for mime sniffing<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

//! A task that takes a URL and streams back the binary data.

use about_loader;
use data_loader;
use file_loader;
use http_loader;
use cookie_storage::CookieStorage;
use cookie;
use mime_classifier::MIMEClassifier;

use net_traits::{ControlMsg, LoadData, LoadResponse, LoadConsumer};
use net_traits::{Metadata, ProgressMsg, ResourceTask, AsyncResponseTarget, ResponseAction};
use net_traits::ProgressMsg::Done;
use util::opts;
use util::task::spawn_named;

use devtools_traits::{DevtoolsControlMsg};
use hyper::header::{ContentType, Header, SetCookie, UserAgent};
use hyper::mime::{Mime, TopLevel, SubLevel};

use std::borrow::ToOwned;
use std::boxed::{self, FnBox};
use std::collections::HashMap;
use std::env;
use std::fs::File;
use std::io::{BufReader, Read};
use std::str::FromStr;
use std::sync::Arc;
use std::sync::mpsc::{channel, Receiver, Sender};

static mut HOST_TABLE: Option<*mut HashMap<String, String>> = None;

pub fn global_init() {
    //TODO: handle bad file path
    let path = match env::var("HOST_FILE") {
        Ok(host_file_path) => host_file_path,
        Err(_) => return,
    };

    let mut file = match File::open(&path) {
        Ok(f) => BufReader::new(f),
        Err(_) => return,
    };

    let mut lines = String::new();
    match file.read_to_string(&mut lines) {
        Ok(_) => (),
        Err(_) => return,
    };

    unsafe {
        let host_table = boxed::into_raw(parse_hostsfile(&lines));
        HOST_TABLE = Some(host_table);
    }
}

pub enum ProgressSender {
    Channel(Sender<ProgressMsg>),
    Listener(Box<AsyncResponseTarget>),
}

impl ProgressSender {
    //XXXjdm return actual error
    pub fn send(&self, msg: ProgressMsg) -> Result<(), ()> {
        match *self {
            ProgressSender::Channel(ref c) => c.send(msg).map_err(|_| ()),
            ProgressSender::Listener(ref b) => {
                let action = match msg {
                    ProgressMsg::Payload(buf) => ResponseAction::DataAvailable(buf),
                    ProgressMsg::Done(status) => ResponseAction::ResponseComplete(status),
                };
                b.invoke_with_listener(action);
                Ok(())
            }
        }
    }
}

/// For use by loaders in responding to a Load message.
pub fn start_sending(start_chan: LoadConsumer, metadata: Metadata) -> ProgressSender {
    start_sending_opt(start_chan, metadata).ok().unwrap()
}

/// For use by loaders in responding to a Load message that allows content sniffing.
pub fn start_sending_sniffed(start_chan: LoadConsumer, metadata: Metadata,
                             classifier: Arc<MIMEClassifier>, partial_body: &Vec<u8>)
                             -> ProgressSender {
    start_sending_sniffed_opt(start_chan, metadata, classifier, partial_body).ok().unwrap()
}

/// For use by loaders in responding to a Load message that allows content sniffing.
pub fn start_sending_sniffed_opt(start_chan: LoadConsumer, mut metadata: Metadata,
                                 classifier: Arc<MIMEClassifier>, partial_body: &Vec<u8>)
                                 -> Result<ProgressSender, ()> {
    if opts::get().sniff_mime_types {
        // TODO: should be calculated in the resource loader, from pull requeset #4094
        let mut nosniff = false;
        let mut check_for_apache_bug = false;

        if let Some(ref headers) = metadata.headers {
            if let Some(ref raw_content_type) = headers.get_raw("content-type") {
                let ref last_raw_content_type = raw_content_type[raw_content_type.len() - 1];
                check_for_apache_bug = last_raw_content_type == b"text/plain"
                                    || last_raw_content_type == b"text/plain; charset=ISO-8859-1"
                                    || last_raw_content_type == b"text/plain; charset=iso-8859-1"
                                    || last_raw_content_type == b"text/plain; charset=UTF-8";
            }
            if let Some(ref raw_content_type_options) = headers.get_raw("content-type-options") {
                for options in raw_content_type_options.iter() {
                    nosniff = nosniff || options == b"nosniff";
                }
            }
        }

        let supplied_type = metadata.content_type.map(|ContentType(Mime(toplevel, sublevel, _))| {
            (format!("{}", toplevel), format!("{}", sublevel))
        });
        metadata.content_type = classifier.classify(nosniff, check_for_apache_bug, &supplied_type, &partial_body).map(|(toplevel, sublevel)| {
            let mime_tp: TopLevel = FromStr::from_str(&toplevel).unwrap();
            let mime_sb: SubLevel = FromStr::from_str(&sublevel).unwrap();
            ContentType(Mime(mime_tp, mime_sb, vec!()))
        });

    }

    start_sending_opt(start_chan, metadata)
}

/// For use by loaders in responding to a Load message.
pub fn start_sending_opt(start_chan: LoadConsumer, metadata: Metadata) -> Result<ProgressSender, ()> {
    match start_chan {
        LoadConsumer::Channel(start_chan) => {
            let (progress_chan, progress_port) = channel();
            let result = start_chan.send(LoadResponse {
                metadata:      metadata,
                progress_port: progress_port,
            });
            match result {
                Ok(_) => Ok(ProgressSender::Channel(progress_chan)),
                Err(_) => Err(())
            }
        }
        LoadConsumer::Listener(target) => {
            target.invoke_with_listener(ResponseAction::HeadersAvailable(metadata));
            Ok(ProgressSender::Listener(target))
        }
    }
}

/// Create a ResourceTask
pub fn new_resource_task(user_agent: Option<String>, devtools_chan: Option<Sender<DevtoolsControlMsg>>) -> ResourceTask {
    let (setup_chan, setup_port) = channel();
    let setup_chan_clone = setup_chan.clone();
    spawn_named("ResourceManager".to_owned(), move || {
        ResourceManager::new(setup_port, user_agent, setup_chan_clone, devtools_chan).start();
    });
    setup_chan
}

pub fn parse_hostsfile(hostsfile_content: &str) -> Box<HashMap<String, String>> {
    let ipv4_regex = regex!(r"^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$");
    let ipv6_regex = regex!(r"^([a-fA-F0-9]{0,4}[:]?){1,8}(/\d{1,3})?$");
    let mut host_table = HashMap::new();
    let lines: Vec<&str> = hostsfile_content.split('\n').collect();

    for line in lines.iter() {
        let ip_host: Vec<&str> = line.trim().split(|c: char| c == ' ' || c == '\t').collect();
        if ip_host.len() > 1 {
            if !ipv4_regex.is_match(ip_host[0]) && !ipv6_regex.is_match(ip_host[0]) { continue; }
            let address = ip_host[0].to_owned();

            for token in ip_host.iter().skip(1) {
                if token.as_bytes()[0] == b'#' {
                    break;
                }
                host_table.insert(token.to_owned().to_string(), address.clone());
            }
        }
    }
    box host_table
}

pub fn replace_hosts(mut load_data: LoadData, host_table: *mut HashMap<String, String>) -> LoadData {
    if let Some(h) = load_data.url.domain_mut() {
        unsafe {
            if let Some(ip) = (*host_table).get(h) {
                *h = ip.clone();
            }
        }
    }
    return load_data;
}

struct ResourceManager {
    from_client: Receiver<ControlMsg>,
    user_agent: Option<String>,
    cookie_storage: CookieStorage,
    resource_task: Sender<ControlMsg>,
    mime_classifier: Arc<MIMEClassifier>,
    devtools_chan: Option<Sender<DevtoolsControlMsg>>
}

impl ResourceManager {
    fn new(from_client: Receiver<ControlMsg>, user_agent: Option<String>,
           resource_task: Sender<ControlMsg>, devtools_channel: Option<Sender<DevtoolsControlMsg>>) -> ResourceManager {
        ResourceManager {
            from_client: from_client,
            user_agent: user_agent,
            cookie_storage: CookieStorage::new(),
            resource_task: resource_task,
            mime_classifier: Arc::new(MIMEClassifier::new()),
            devtools_chan: devtools_channel
        }
    }
}

impl ResourceManager {
    fn start(&mut self) {
        loop {
            match self.from_client.recv().unwrap() {
              ControlMsg::Load(load_data, consumer) => {
                self.load(load_data, consumer)
              }
              ControlMsg::SetCookiesForUrl(request, cookie_list, source) => {
                let header = Header::parse_header(&[cookie_list.into_bytes()]);
                if let Some(SetCookie(cookies)) = header {
                  for bare_cookie in cookies.into_iter() {
                    if let Some(cookie) = cookie::Cookie::new_wrapped(bare_cookie, &request, source) {
                      self.cookie_storage.push(cookie, source);
                    }
                  }
                }
              }
              ControlMsg::GetCookiesForUrl(url, consumer, source) => {
                consumer.send(self.cookie_storage.cookies_for_url(&url, source)).unwrap();
              }
              ControlMsg::Exit => {
                break
              }
            }
        }
    }

    fn load(&mut self, mut load_data: LoadData, consumer: LoadConsumer) {
        unsafe {
            if let Some(host_table) = HOST_TABLE {
                load_data = replace_hosts(load_data, host_table);
            }
        }

        self.user_agent.as_ref().map(|ua| load_data.headers.set(UserAgent(ua.clone())));

        fn from_factory(factory: fn(LoadData, LoadConsumer, Arc<MIMEClassifier>))
                        -> Box<FnBox(LoadData, LoadConsumer, Arc<MIMEClassifier>) + Send> {
            box move |load_data, senders, classifier| {
                factory(load_data, senders, classifier)
            }
        }

        let loader = match &*load_data.url.scheme {
            "file" => from_factory(file_loader::factory),
            "http" | "https" | "view-source" => http_loader::factory(self.resource_task.clone(), self.devtools_chan.clone()),
            "data" => from_factory(data_loader::factory),
            "about" => from_factory(about_loader::factory),
            _ => {
                debug!("resource_task: no loader for scheme {}", load_data.url.scheme);
                start_sending(consumer, Metadata::default(load_data.url))
                    .send(ProgressMsg::Done(Err("no loader for scheme".to_string()))).unwrap();
                return
            }
        };
        debug!("resource_task: loading url: {}", load_data.url.serialize());

        loader.call_box((load_data, consumer, self.mime_classifier.clone()));
    }
}
<|endoftext|>


<commit_before>// The example only works on Linux since Termion does not yet support
// Windows: https://gitlab.redox-os.org/redox-os/termion/-/issues/103
// The precise library doesn't matter much, so feel free to send a PR
// if there is a library with good Windows support.

fn main() -> Result<(), std::io::Error> {
    #[cfg(not(unix))]
    panic!("Sorry, this example currently only works on Unix!");

    #[cfg(unix)]
    unix_only::main()
}

#[cfg(unix)]
mod unix_only {
    use std::io::{self, Write};
    use termion::event::Key;
    use termion::input::TermRead;
    use termion::raw::{IntoRawMode, RawTerminal};
    use termion::screen::AlternateScreen;
    use termion::{color, cursor, style};
    use textwrap::core::WrapAlgorithm::{FirstFit, OptimalFit};
    use textwrap::{wrap, HyphenSplitter, NoHyphenation, Options, WordSplitter};

    #[cfg(feature = "hyphenation")]
    use hyphenation::{Language, Load, Standard};

    fn draw_margins(
        row: u16,
        col: u16,
        line_width: u16,
        left: char,
        right: char,
        stdout: &mut RawTerminal<io::Stdout>,
    ) -> Result<(), io::Error> {
        write!(
            stdout,
            "{}{}{}{}",
            cursor::Goto(col - 1, row),
            color::Fg(color::Red),
            left,
            color::Fg(color::Reset),
        )?;
        write!(
            stdout,
            "{}{}{}{}",
            cursor::Goto(col + line_width, row),
            color::Fg(color::Red),
            right,
            color::Fg(color::Reset),
        )?;

        Ok(())
    }

    fn draw_text<'a>(
        text: &str,
        options: &Options<'a>,
        splitter_label: &str,
        stdout: &mut RawTerminal<io::Stdout>,
    ) -> Result<(), io::Error> {
        let mut left_row: u16 = 1;
        let left_col: u16 = 3;

        write!(stdout, "{}", termion::clear::All)?;
        write!(
            stdout,
            "{}{}Options:{}",
            cursor::Goto(left_col, left_row),
            style::Bold,
            style::Reset,
        )?;
        left_row += 1;

        write!(
            stdout,
            "{}- width: {}{}{} (use ← and → to change)",
            cursor::Goto(left_col, left_row),
            style::Bold,
            options.width,
            style::Reset,
        )?;
        left_row += 1;

        write!(
            stdout,
            "{}- break_words: {}{:?}{} (toggle with Ctrl-b)",
            cursor::Goto(left_col, left_row),
            style::Bold,
            options.break_words,
            style::Reset,
        )?;
        left_row += 1;

        write!(
            stdout,
            "{}- splitter: {}{}{} (cycle with Ctrl-s)",
            cursor::Goto(left_col, left_row),
            style::Bold,
            splitter_label,
            style::Reset,
        )?;
        left_row += 1;

        write!(
            stdout,
            "{}- algorithm: {}{:?}{} (toggle with Ctrl-o)",
            cursor::Goto(left_col, left_row),
            style::Bold,
            options.wrap_algorithm,
            style::Reset,
        )?;
        left_row += 1;

        let now = std::time::Instant::now();
        let mut lines = wrap(text, options);
        let elapsed = now.elapsed();

        let right_col: u16 = 55;
        let mut right_row: u16 = 1;
        write!(
            stdout,
            "{}{}Performance:{}",
            cursor::Goto(right_col, right_row),
            style::Bold,
            style::Reset,
        )?;
        right_row += 1;

        write!(
            stdout,
            "{}- words: {}{}{}",
            cursor::Goto(right_col, right_row),
            style::Bold,
            text.split_whitespace().count(),
            style::Reset,
        )?;
        right_row += 1;

        write!(
            stdout,
            "{}- characters: {}{}{}",
            cursor::Goto(right_col, right_row),
            style::Bold,
            text.chars().count(),
            style::Reset,
        )?;
        right_row += 1;

        write!(
            stdout,
            "{}- latency: {}{} usec{}",
            cursor::Goto(right_col, right_row),
            style::Bold,
            elapsed.as_micros(),
            style::Reset,
        )?;

        // Empty line.
        left_row += 1;

        if let Some(line) = lines.last_mut() {
            let trailing_whitespace = &text[text.trim_end_matches(' ').len()..];
            if !trailing_whitespace.is_empty() {
                // Trailing whitespace is discarded by
                // `textwrap::wrap`. We reinsert it here. If multiple
                // spaces are added, this can overflow the margins
                // which look a bit odd. Handling this would require
                // some more tinkering...
                *line = format!("{}{}", line, trailing_whitespace).into();
            } else if line.ends_with('\n') {
                // If `text` ends with a newline, the final wrapped line
                // contains this newline. This will in turn leave the
                // cursor hanging in the middle of the line. Pushing an
                // extra empty line fixes this.
                lines.push("".into());
            }
        } else {
            // No lines -> we add an empty line so we have a place
            // where we can display the cursor.
            lines.push("".into());
        }

        // Draw margins above and below the wrapped text. We draw the
        // margin before the text so that 1) the text can overwrite
        // the margin if `break_words` is `false` and `width` is very
        // small and 2) so the cursor remains at the end of the last
        // line of text.
        draw_margins(left_row, left_col, options.width as u16, '┌', '┐', stdout)?;
        left_row += 1;
        let final_row = left_row + lines.len() as u16;
        draw_margins(final_row, left_col, options.width as u16, '└', '┘', stdout)?;

        let (_, rows) = termion::terminal_size()?;
        write!(stdout, "{}", cursor::Show)?;
        for line in lines {
            if left_row > rows {
                // The text does not fits on the terminal -- we hide
                // the cursor since it's supposed to be "below" the
                // bottom of the terminal.
                write!(stdout, "{}", cursor::Hide)?;
                break;
            }
            draw_margins(left_row, left_col, options.width as u16, '│', '│', stdout)?;
            write!(stdout, "{}{}", cursor::Goto(left_col, left_row), line)?;
            left_row += 1;
        }

        stdout.flush()
    }

    pub fn main() -> Result<(), io::Error> {
        let mut labels = vec![
            String::from("HyphenSplitter"),
            String::from("NoHyphenation"),
        ];

        let mut splitters: Vec<Box<dyn WordSplitter>> =
            vec![Box::new(HyphenSplitter), Box::new(NoHyphenation)];

        // If you like, you can download more dictionaries from
        // https://github.com/tapeinosyne/hyphenation/tree/master/dictionaries
        // Place the dictionaries in the examples/ directory. Here we
        // just load the embedded en-us dictionary.
        #[cfg(feature = "hyphenation")]
        for lang in &[Language::EnglishUS] {
            let dictionary = Standard::from_embedded(*lang).or_else(|_| {
                let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
                    .join("examples")
                    .join(format!("{}.standard.bincode", lang.code()));
                Standard::from_path(*lang, &path)
            });

            if let Ok(dict) = dictionary {
                labels.push(format!("{} hyphenation", lang.code()));
                splitters.push(Box::new(dict));
            }
        }

        let mut label = labels.pop().unwrap();
        let mut options: Options = Options::new(35).splitter(Box::new(HyphenSplitter));
        options.break_words = false;
        options.splitter = splitters.pop().unwrap();

        let mut idx_iter = (0..splitters.len()).collect::<Vec<_>>().into_iter().cycle();

        let mut text = String::from(
            "Welcome to the interactive demo! The following is from The \
            Emperor’s New Clothes by Hans Christian Andersen. You can edit the \
            text!\n\n\
            Many years ago there was an Emperor, who was so excessively fond \
            of new clothes that he spent all his money on them. He cared \
            nothing about his soldiers, nor for the theatre, nor for driving \
            in the woods except for the sake of showing off his new clothes. \
            He had a costume for every hour in the day, and instead of saying, \
            as one does about any other king or emperor, ‘He is in his council \
            chamber,’ here one always said, ‘The Emperor is in his \
            dressing-room.’",
        );

        let stdin = io::stdin();
        let mut screen = AlternateScreen::from(io::stdout().into_raw_mode()?);
        write!(screen, "{}", cursor::BlinkingUnderline)?;
        draw_text(&text, &options, &label, &mut screen)?;

        for c in stdin.keys() {
            match c? {
                Key::Esc | Key::Ctrl('c') => break,
                Key::Left => options.width = options.width.saturating_sub(1),
                Key::Right => options.width = options.width.saturating_add(1),
                Key::Ctrl('b') => options.break_words = !options.break_words,
                Key::Ctrl('o') => {
                    options.wrap_algorithm = match options.wrap_algorithm {
                        OptimalFit => FirstFit,
                        FirstFit => OptimalFit,
                    }
                }
                Key::Ctrl('s') => {
                    let idx = idx_iter.next().unwrap();
                    std::mem::swap(&mut options.splitter, &mut splitters[idx]);
                    std::mem::swap(&mut label, &mut labels[idx]);
                }
                Key::Char(c) => text.push(c),
                Key::Backspace => {
                    text.pop();
                }
                _ => {}
            }

            draw_text(&text, &options, &label, &mut screen)?;
        }

        // TODO: change to cursor::DefaultStyle if
        // https://github.com/redox-os/termion/pull/157 is merged.
        screen.write_all(b"\x1b[0 q")?;
        screen.flush()
    }
}
<commit_msg>Show build type (debug/release) in interactive demo<commit_after>// The example only works on Linux since Termion does not yet support
// Windows: https://gitlab.redox-os.org/redox-os/termion/-/issues/103
// The precise library doesn't matter much, so feel free to send a PR
// if there is a library with good Windows support.

fn main() -> Result<(), std::io::Error> {
    #[cfg(not(unix))]
    panic!("Sorry, this example currently only works on Unix!");

    #[cfg(unix)]
    unix_only::main()
}

#[cfg(unix)]
mod unix_only {
    use std::io::{self, Write};
    use termion::event::Key;
    use termion::input::TermRead;
    use termion::raw::{IntoRawMode, RawTerminal};
    use termion::screen::AlternateScreen;
    use termion::{color, cursor, style};
    use textwrap::core::WrapAlgorithm::{FirstFit, OptimalFit};
    use textwrap::{wrap, HyphenSplitter, NoHyphenation, Options, WordSplitter};

    #[cfg(feature = "hyphenation")]
    use hyphenation::{Language, Load, Standard};

    fn draw_margins(
        row: u16,
        col: u16,
        line_width: u16,
        left: char,
        right: char,
        stdout: &mut RawTerminal<io::Stdout>,
    ) -> Result<(), io::Error> {
        write!(
            stdout,
            "{}{}{}{}",
            cursor::Goto(col - 1, row),
            color::Fg(color::Red),
            left,
            color::Fg(color::Reset),
        )?;
        write!(
            stdout,
            "{}{}{}{}",
            cursor::Goto(col + line_width, row),
            color::Fg(color::Red),
            right,
            color::Fg(color::Reset),
        )?;

        Ok(())
    }

    fn draw_text<'a>(
        text: &str,
        options: &Options<'a>,
        splitter_label: &str,
        stdout: &mut RawTerminal<io::Stdout>,
    ) -> Result<(), io::Error> {
        let mut left_row: u16 = 1;
        let left_col: u16 = 3;

        write!(stdout, "{}", termion::clear::All)?;
        write!(
            stdout,
            "{}{}Options:{}",
            cursor::Goto(left_col, left_row),
            style::Bold,
            style::Reset,
        )?;
        left_row += 1;

        write!(
            stdout,
            "{}- width: {}{}{} (use ← and → to change)",
            cursor::Goto(left_col, left_row),
            style::Bold,
            options.width,
            style::Reset,
        )?;
        left_row += 1;

        write!(
            stdout,
            "{}- break_words: {}{:?}{} (toggle with Ctrl-b)",
            cursor::Goto(left_col, left_row),
            style::Bold,
            options.break_words,
            style::Reset,
        )?;
        left_row += 1;

        write!(
            stdout,
            "{}- splitter: {}{}{} (cycle with Ctrl-s)",
            cursor::Goto(left_col, left_row),
            style::Bold,
            splitter_label,
            style::Reset,
        )?;
        left_row += 1;

        write!(
            stdout,
            "{}- algorithm: {}{:?}{} (toggle with Ctrl-o)",
            cursor::Goto(left_col, left_row),
            style::Bold,
            options.wrap_algorithm,
            style::Reset,
        )?;
        left_row += 1;

        let now = std::time::Instant::now();
        let mut lines = wrap(text, options);
        let elapsed = now.elapsed();

        let right_col: u16 = 55;
        let mut right_row: u16 = 1;
        write!(
            stdout,
            "{}{}Performance:{}",
            cursor::Goto(right_col, right_row),
            style::Bold,
            style::Reset,
        )?;
        right_row += 1;

        write!(
            stdout,
            "{}- build: {}{}{}",
            cursor::Goto(right_col, right_row),
            style::Bold,
            if cfg!(debug_assertions) {
                "debug"
            } else {
                "release"
            },
            style::Reset,
        )?;
        right_row += 1;

        write!(
            stdout,
            "{}- words: {}{}{}",
            cursor::Goto(right_col, right_row),
            style::Bold,
            text.split_whitespace().count(),
            style::Reset,
        )?;
        right_row += 1;

        write!(
            stdout,
            "{}- characters: {}{}{}",
            cursor::Goto(right_col, right_row),
            style::Bold,
            text.chars().count(),
            style::Reset,
        )?;
        right_row += 1;

        write!(
            stdout,
            "{}- latency: {}{} usec{}",
            cursor::Goto(right_col, right_row),
            style::Bold,
            elapsed.as_micros(),
            style::Reset,
        )?;

        // Empty line.
        left_row += 1;

        if let Some(line) = lines.last_mut() {
            let trailing_whitespace = &text[text.trim_end_matches(' ').len()..];
            if !trailing_whitespace.is_empty() {
                // Trailing whitespace is discarded by
                // `textwrap::wrap`. We reinsert it here. If multiple
                // spaces are added, this can overflow the margins
                // which look a bit odd. Handling this would require
                // some more tinkering...
                *line = format!("{}{}", line, trailing_whitespace).into();
            } else if line.ends_with('\n') {
                // If `text` ends with a newline, the final wrapped line
                // contains this newline. This will in turn leave the
                // cursor hanging in the middle of the line. Pushing an
                // extra empty line fixes this.
                lines.push("".into());
            }
        } else {
            // No lines -> we add an empty line so we have a place
            // where we can display the cursor.
            lines.push("".into());
        }

        // Draw margins above and below the wrapped text. We draw the
        // margin before the text so that 1) the text can overwrite
        // the margin if `break_words` is `false` and `width` is very
        // small and 2) so the cursor remains at the end of the last
        // line of text.
        draw_margins(left_row, left_col, options.width as u16, '┌', '┐', stdout)?;
        left_row += 1;
        let final_row = left_row + lines.len() as u16;
        draw_margins(final_row, left_col, options.width as u16, '└', '┘', stdout)?;

        let (_, rows) = termion::terminal_size()?;
        write!(stdout, "{}", cursor::Show)?;
        for line in lines {
            if left_row > rows {
                // The text does not fits on the terminal -- we hide
                // the cursor since it's supposed to be "below" the
                // bottom of the terminal.
                write!(stdout, "{}", cursor::Hide)?;
                break;
            }
            draw_margins(left_row, left_col, options.width as u16, '│', '│', stdout)?;
            write!(stdout, "{}{}", cursor::Goto(left_col, left_row), line)?;
            left_row += 1;
        }

        stdout.flush()
    }

    pub fn main() -> Result<(), io::Error> {
        let mut labels = vec![
            String::from("HyphenSplitter"),
            String::from("NoHyphenation"),
        ];

        let mut splitters: Vec<Box<dyn WordSplitter>> =
            vec![Box::new(HyphenSplitter), Box::new(NoHyphenation)];

        // If you like, you can download more dictionaries from
        // https://github.com/tapeinosyne/hyphenation/tree/master/dictionaries
        // Place the dictionaries in the examples/ directory. Here we
        // just load the embedded en-us dictionary.
        #[cfg(feature = "hyphenation")]
        for lang in &[Language::EnglishUS] {
            let dictionary = Standard::from_embedded(*lang).or_else(|_| {
                let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
                    .join("examples")
                    .join(format!("{}.standard.bincode", lang.code()));
                Standard::from_path(*lang, &path)
            });

            if let Ok(dict) = dictionary {
                labels.push(format!("{} hyphenation", lang.code()));
                splitters.push(Box::new(dict));
            }
        }

        let mut label = labels.pop().unwrap();
        let mut options: Options = Options::new(35).splitter(Box::new(HyphenSplitter));
        options.break_words = false;
        options.splitter = splitters.pop().unwrap();

        let mut idx_iter = (0..splitters.len()).collect::<Vec<_>>().into_iter().cycle();

        let mut text = String::from(
            "Welcome to the interactive demo! The following is from The \
            Emperor’s New Clothes by Hans Christian Andersen. You can edit the \
            text!\n\n\
            Many years ago there was an Emperor, who was so excessively fond \
            of new clothes that he spent all his money on them. He cared \
            nothing about his soldiers, nor for the theatre, nor for driving \
            in the woods except for the sake of showing off his new clothes. \
            He had a costume for every hour in the day, and instead of saying, \
            as one does about any other king or emperor, ‘He is in his council \
            chamber,’ here one always said, ‘The Emperor is in his \
            dressing-room.’",
        );

        let stdin = io::stdin();
        let mut screen = AlternateScreen::from(io::stdout().into_raw_mode()?);
        write!(screen, "{}", cursor::BlinkingUnderline)?;
        draw_text(&text, &options, &label, &mut screen)?;

        for c in stdin.keys() {
            match c? {
                Key::Esc | Key::Ctrl('c') => break,
                Key::Left => options.width = options.width.saturating_sub(1),
                Key::Right => options.width = options.width.saturating_add(1),
                Key::Ctrl('b') => options.break_words = !options.break_words,
                Key::Ctrl('o') => {
                    options.wrap_algorithm = match options.wrap_algorithm {
                        OptimalFit => FirstFit,
                        FirstFit => OptimalFit,
                    }
                }
                Key::Ctrl('s') => {
                    let idx = idx_iter.next().unwrap();
                    std::mem::swap(&mut options.splitter, &mut splitters[idx]);
                    std::mem::swap(&mut label, &mut labels[idx]);
                }
                Key::Char(c) => text.push(c),
                Key::Backspace => {
                    text.pop();
                }
                _ => {}
            }

            draw_text(&text, &options, &label, &mut screen)?;
        }

        // TODO: change to cursor::DefaultStyle if
        // https://github.com/redox-os/termion/pull/157 is merged.
        screen.write_all(b"\x1b[0 q")?;
        screen.flush()
    }
}
<|endoftext|>


<commit_before>extern crate image;
extern crate texture_packer;

use std::path::Path;
use std::fs::File;

use texture_packer::texture::Texture;
use texture_packer::{ TexturePacker, TexturePackerConfig };
use texture_packer::importer::ImageImporter;
use texture_packer::exporter::ImageExporter;

const MAX_IMAGE_WIDTH: u32 = 400;
const MAX_IMAGE_HEIGHT: u32 = 400;

fn main() {
    let mut config = TexturePackerConfig::default();
    config.max_width = MAX_IMAGE_WIDTH;
    config.max_height = MAX_IMAGE_HEIGHT;
    config.allow_rotation = false;
    config.texture_outlines = true;
    config.border_padding = 2;

    let ref mut texture_packer = TexturePacker::new_skyline(config);

    for i in 1 .. 11 {
        let file = format!("{}.png", i);
        let ref path = ["./examples/assets/", &file[..]].concat();
        let ref path = Path::new(path);
        let texture = ImageImporter::import_from_file(path).unwrap();

        texture_packer.pack_own(file, texture);
    }

    let image = ImageExporter::export(texture_packer).unwrap();
    let path = "./examples/output/skyline-packer-output.png";
    let ref path = Path::new(path);
    let ref mut file = File::create(path).unwrap();

    println!("{} x {}", texture_packer.width(), texture_packer.height());
    image.save(file, image::PNG).unwrap();
}
<commit_msg>Simplify the sample code<commit_after>extern crate image;
extern crate texture_packer;

use std::path::Path;
use std::fs::File;

use texture_packer::texture::Texture;
use texture_packer::{TexturePacker, TexturePackerConfig};
use texture_packer::importer::ImageImporter;
use texture_packer::exporter::ImageExporter;

fn main() {
    //
    // Perform texture packing
    //
    let config = TexturePackerConfig {
        max_width: 400,
        max_height: 400,
        allow_rotation: false,
        texture_outlines: true,
        border_padding: 2,
        ..Default::default()
    };

    let mut packer = TexturePacker::new_skyline(config);

    for i in 1..11 {
        let name = format!("{}.png", i);
        let path = format!("examples/assets/{}", name);
        let path = Path::new(&path);
        let texture = ImageImporter::import_from_file(&path).unwrap();

        packer.pack_own(name, texture);
    }

    //
    // Print the information
    //
    println!("Dimensions : {}x{}", packer.width(), packer.height());
    println!("");
    for (name, frame) in packer.get_frames() {
        println!("  {:7} : {:?}", name, frame.frame);
    }

    //
    // Save the result
    //
    let exporter = ImageExporter::export(&packer).unwrap();
    let mut file = File::create("examples/output/skyline-packer-output.png").unwrap();
    exporter.save(&mut file, image::PNG).unwrap();
}
<|endoftext|>


<commit_before>use std::fs::File;
use std::rc::Rc;
use std::io::prelude::*;
use std::path::Path;
use std::fs::DirBuilder;

use bincode::rustc_serialize::{encode_into};
use bincode::SizeLimit;

pub struct Database {
    path: String,
    metadata_buf: Option<Rc<File>>,
    metadata: DatabaseMetadata
}

#[derive(RustcEncodable, RustcDecodable, PartialEq, Clone)]
struct DatabaseMetadata {
    name: String,
    tables: u16
}

impl Database {
    pub fn new(p: &str, name: &str) -> Self {
        Database {
            path: p.to_string(),
            metadata_buf: None,
            metadata: DatabaseMetadata {
                name: name.to_string(),
                tables: 0
            }
        }
    }

    pub fn create(&mut self) {
        DirBuilder::new()
            .recursive(true)
            .create(&self.path).unwrap();

        let mut f: File = File::create(Path::new(&self.path).join(&self.metadata.name)).unwrap();

        let limit = SizeLimit::Bounded(20);

        encode_into(&Some(self.metadata.clone()), &mut f, limit).unwrap();

        self.metadata_buf = Some(Rc::new(f));
        info!("Added database: {}", self.metadata.name);
    }

    #[allow(dead_code)]
    pub fn load(&mut self) {
        let mut f = File::open("/tmp/database_test".to_string() + &self.metadata.name).unwrap();
        let mut name = String::new();
        f.read_to_string(&mut name).unwrap();

        if name == self.metadata.name.to_string() {
            self.metadata_buf = Some(Rc::new(f));
        }
    }
}

#[cfg(test)]
mod test {
    use super::Database;
    use std::fs::{File, metadata};

    #[test]
    fn create_db() {
        let path = "/tmp/test1/";
        let mut db = Database::new(&path, "test_db");
        db.create();

        assert!(metadata(&path).is_ok());
        assert!(metadata(&path).unwrap().is_dir());
        assert!(File::open("/tmp/test1/test_db").is_ok());
    }
}
<commit_msg>more fixes for load; not working<commit_after>use std::fs::File;
use std::rc::Rc;
use std::io::prelude::*;
use std::path::Path;
use std::fs::DirBuilder;

use bincode::rustc_serialize::{encode_into, decode_from, DecodingError};
use bincode::SizeLimit;

pub struct Database {
    path: String,
    metadata_buf: Option<Rc<File>>,
     metadata: DatabaseMetadata
}

#[derive(RustcEncodable, RustcDecodable, PartialEq, Clone)]
 struct DatabaseMetadata {
    name: String,
    tables: u8
}

impl Database {
    pub fn new(p: &str, name: &str) -> Self {
        Database {
            path: p.to_string(),
            metadata_buf: None,
            metadata: DatabaseMetadata {
                name: name.to_string(),
                tables: 0
            }
        }
    }

    pub fn create(&mut self) {
        DirBuilder::new()
            .recursive(true)
            .create(&self.path).unwrap();

        let mut f: File = File::create(Path::new(&self.path).join(&self.metadata.name)).unwrap();

        let limit = SizeLimit::Infinite;

        encode_into(&Some(self.metadata.clone()), &mut f, limit).unwrap();

        self.metadata_buf = Some(Rc::new(f));
        info!("Added database: {}", self.metadata.name);
    }

    #[warn(dead_code)]
    pub fn load(&mut self) -> Result<(), DecodingError>{
        let mut f = File::open(Path::new(&self.path).join(&self.metadata.name)).unwrap();
        let limit = SizeLimit::Infinite;
        let tmp: DatabaseMetadata = try!(decode_from(&mut f, limit));

        if tmp.name == self.metadata.name.to_string() {
            self.metadata_buf = Some(Rc::new(f));
        }
        Ok(())
    }

    pub fn get_name(&self) -> String {
        self.metadata.name.clone()
    }
}

#[cfg(test)]
mod test {
    use super::Database;
    use std::fs::{File, metadata};
    use std::path::Path;

    #[test]
    fn create_db() {
        let path = "/tmp/test1/";
        let name = "test_db";
        let mut db = Database::new(&path, &name);
        db.create();

        assert!(metadata(&path).is_ok());
        assert!(metadata(&path).unwrap().is_dir());
        assert!(File::open(Path::new(&path).join(&name)).is_ok());
    }

    #[test]
    fn load_db() {
        #![ignore]
        let path = "/tmp/test1/";
        let name = "test_db2";
        let mut db = Database::new(&path, &name);
        db.create();

        let mut db2 = Database::new(&path, &name);
        if let Err(e) = db2.load() {
            println!("{:?}", e);
            assert!(false);
        }

        assert_eq!(db.get_name(), db2.get_name());
    }
}
<|endoftext|>


<commit_before>//
//  audio_io.rs
//
//  Created by Mitchell Nordine at 06:01AM on June 17, 2014.
//
//

//! A real-time soundstream demo that smoothly copies input
//! from a microphone straight to the output. If <Space> is
//! pressed, the SoundStream thread will begin to calculate
//! and print the samplerate to demonstrate the inter-task
//! event handling.
//!
//! Note: Beware of feedback!

#![feature(globs)]

extern crate piston;

use piston::{
    InteractiveEvent,
    keyboard,
    AssetStore,
    Game,
    GameWindow,
    GameWindowSDL2,
    GameWindowSettings,
    KeyPress,
    KeyPressArgs,
    SoundStream,
    soundstreamer
};

//------------------------------

static SAMPLE_RATE: f64 = 44100f64;
static NUM_FRAMES: u32 = 1024u32;
static NUM_CHANNELS: i32 = 2i32;

// Structs
//------------------------------

/// Main application struct.
pub struct App {
    /// Channel for sending information to the audio stream.
    stream_chan: Option<Sender<InteractiveEvent>> // Channel for sending Events.
}

/// The audio is non-blocking and needs it's own struct.
pub struct AppSoundStream {
    /// Channel for receiving game events from main game stream.
    chan: Option<Receiver<InteractiveEvent>>, // Channel for receiving Events.
    is_exit: bool, // Trigger for closing the stream.
    is_print: bool, // Toggle for printing the sample_rate.
    buffer: Vec<f32> // Buffer for passing input to output.
}

// Game Method Implementations
//------------------------------

impl Game for App {

    /// Setup / load the app stuff ready for the main loop.
    /// If using a SoundStream, it must be created within this method.
    fn load(&mut self, asset_store: &mut AssetStore) {

        // Create a channel for communicating events with the soundstream.
        // Note: this channel is used for sending InteractiveEvents, but
        // the same technique could be used here to create custom channels
        // that can safely send any kind of unique data.
        let (send, recv) = channel();
        self.stream_chan = Some(send);

        // Create the soundstream on it's own thread for non-blocking, real-time audio.
        // "soundstreamer" will setup and iterate soundstream using portaudio.
        spawn(proc() {
            let mut soundstream = AppSoundStream::new(Some(recv));
            soundstreamer(SAMPLE_RATE, NUM_FRAMES, NUM_CHANNELS, &mut soundstream);
        });

    }

    /// Keypress callback.
    fn key_press(&mut self, args: &KeyPressArgs) {
        println!("Game thread key: {}", args.key);
    }

    /// Specify the event sending channel. This must be done if we wish
    /// to send interactive events to the SoundStream.
    fn get_event_sender(&self) -> Option<Sender<InteractiveEvent>> {
        self.stream_chan.clone()
    }

}

impl Drop for App {
    /// Tell the soundstream to exit when App is destroyed.
    fn drop(&mut self) {
        let chan = self.stream_chan.clone();
        match chan {
            Some(sender) => sender.send(KeyPress(KeyPressArgs { key: keyboard::Escape })),
            None => ()
        }
    }
}

impl App {
    /// Creates a new application.
    pub fn new() -> App {
        App {
            stream_chan: None
        }
    }
}

// SoundStream Method Implementations
//------------------------------

impl SoundStream for AppSoundStream {

    /// Load (called prior to main soundstream loop).
    fn load(&mut self) {
        println!("Press <Spacebar> to start/stop printing the real-time sample rate.");
    }

    /// Update (gets called prior to audio_in/audio_out).
    fn update(&mut self, dt: u64) {
        if self.is_print {
            let dtsec: f64 = dt as f64 / 1000000000f64;
            println!("Real-time sample rate: {}", (1f64 / dtsec) * NUM_FRAMES as f64);
        }
    }

    /// AudioInput
    fn audio_in(&mut self, input: &Vec<f32>, num_frames: u32, num_channels: i32) {
        self.buffer = input.clone();
    }

    /// AudioOutput
    fn audio_out(&mut self, output: &mut Vec<f32>, num_frames: u32, num_channels: i32) {
        *output = self.buffer.clone()
    }

    /// KeyPress
    fn key_press(&mut self, args: &KeyPressArgs) {
        println!("Soundstream thread key: {}", args.key);
        if args.key == keyboard::Space {
            let b_print = if self.is_print { false } else { true };
            self.is_print = b_print;
        }
        if args.key == keyboard::Escape {
            self.is_exit = true;
        }
    }

    /// Retrieve Events for callback (i.e. mouse, keyboard).
    fn check_for_event(&self) -> Option<InteractiveEvent> {
        match self.chan {
            Some(ref receiver) => match receiver.try_recv() {
                Ok(event) => Some(event),
                Err(_) => None
            },
            None => None
        }
    }

    /// Setup the exit condition (is checked once per buffer).
    fn exit(&self) -> bool { self.is_exit }

}

impl AppSoundStream {
    /// AppSoundStream constructor.
    pub fn new(recv: Option<Receiver<InteractiveEvent>>) -> AppSoundStream {
        AppSoundStream {
            chan: recv,
            is_exit: false,
            is_print: false,
            buffer: vec![]
        }
    }
}

// Main
//------------------------------

#[start]
fn start(argc: int, argv: **u8) -> int {
    // Run gui on the main thread.
    native::start(argc, argv, main)
}

fn main() {
    let mut window: GameWindowSDL2 = GameWindow::new(
        GameWindowSettings {
            title: "soundstream".to_string(),
            size: [300, 300],
            fullscreen: false,
            exit_on_esc: true,
            background_color: [0.1, 0.1, 0.1, 0.1],
        }
    );

    let mut asset_store = AssetStore::from_folder("assets");
    let mut app = App::new();
    app.run(&mut window, &mut asset_store);
}

//------------------------------
<commit_msg>Fixed soundstream example header<commit_after>//! soundstream.rs
//!
//! A real-time soundstream demo that smoothly copies input
//! from a microphone straight to the output. If <Space> is
//! pressed, the SoundStream thread will begin to calculate
//! and print the samplerate to demonstrate the inter-task
//! event handling.
//!
//! Note: Beware of feedback!

#![feature(globs)]

extern crate piston;

use piston::{
    InteractiveEvent,
    keyboard,
    AssetStore,
    Game,
    GameWindow,
    GameWindowSDL2,
    GameWindowSettings,
    KeyPress,
    KeyPressArgs,
    SoundStream,
    soundstreamer
};

//------------------------------

static SAMPLE_RATE: f64 = 44100f64;
static NUM_FRAMES: u32 = 1024u32;
static NUM_CHANNELS: i32 = 2i32;

// Structs
//------------------------------

/// Main application struct.
pub struct App {
    /// Channel for sending information to the audio stream.
    stream_chan: Option<Sender<InteractiveEvent>> // Channel for sending Events.
}

/// The audio is non-blocking and needs it's own struct.
pub struct AppSoundStream {
    /// Channel for receiving game events from main game stream.
    chan: Option<Receiver<InteractiveEvent>>, // Channel for receiving Events.
    is_exit: bool, // Trigger for closing the stream.
    is_print: bool, // Toggle for printing the sample_rate.
    buffer: Vec<f32> // Buffer for passing input to output.
}

// Game Method Implementations
//------------------------------

impl Game for App {

    /// Setup / load the app stuff ready for the main loop.
    /// If using a SoundStream, it must be created within this method.
    fn load(&mut self, asset_store: &mut AssetStore) {

        // Create a channel for communicating events with the soundstream.
        // Note: this channel is used for sending InteractiveEvents, but
        // the same technique could be used here to create custom channels
        // that can safely send any kind of unique data.
        let (send, recv) = channel();
        self.stream_chan = Some(send);

        // Create the soundstream on it's own thread for non-blocking, real-time audio.
        // "soundstreamer" will setup and iterate soundstream using portaudio.
        spawn(proc() {
            let mut soundstream = AppSoundStream::new(Some(recv));
            soundstreamer(SAMPLE_RATE, NUM_FRAMES, NUM_CHANNELS, &mut soundstream);
        });

    }

    /// Keypress callback.
    fn key_press(&mut self, args: &KeyPressArgs) {
        println!("Game thread key: {}", args.key);
    }

    /// Specify the event sending channel. This must be done if we wish
    /// to send interactive events to the SoundStream.
    fn get_event_sender(&self) -> Option<Sender<InteractiveEvent>> {
        self.stream_chan.clone()
    }

}

impl Drop for App {
    /// Tell the soundstream to exit when App is destroyed.
    fn drop(&mut self) {
        let chan = self.stream_chan.clone();
        match chan {
            Some(sender) => sender.send(KeyPress(KeyPressArgs { key: keyboard::Escape })),
            None => ()
        }
    }
}

impl App {
    /// Creates a new application.
    pub fn new() -> App {
        App {
            stream_chan: None
        }
    }
}

// SoundStream Method Implementations
//------------------------------

impl SoundStream for AppSoundStream {

    /// Load (called prior to main soundstream loop).
    fn load(&mut self) {
        println!("Press <Spacebar> to start/stop printing the real-time sample rate.");
    }

    /// Update (gets called prior to audio_in/audio_out).
    fn update(&mut self, dt: u64) {
        if self.is_print {
            let dtsec: f64 = dt as f64 / 1000000000f64;
            println!("Real-time sample rate: {}", (1f64 / dtsec) * NUM_FRAMES as f64);
        }
    }

    /// AudioInput
    fn audio_in(&mut self, input: &Vec<f32>, num_frames: u32, num_channels: i32) {
        self.buffer = input.clone();
    }

    /// AudioOutput
    fn audio_out(&mut self, output: &mut Vec<f32>, num_frames: u32, num_channels: i32) {
        *output = self.buffer.clone()
    }

    /// KeyPress
    fn key_press(&mut self, args: &KeyPressArgs) {
        println!("Soundstream thread key: {}", args.key);
        if args.key == keyboard::Space {
            let b_print = if self.is_print { false } else { true };
            self.is_print = b_print;
        }
        if args.key == keyboard::Escape {
            self.is_exit = true;
        }
    }

    /// Retrieve Events for callback (i.e. mouse, keyboard).
    fn check_for_event(&self) -> Option<InteractiveEvent> {
        match self.chan {
            Some(ref receiver) => match receiver.try_recv() {
                Ok(event) => Some(event),
                Err(_) => None
            },
            None => None
        }
    }

    /// Setup the exit condition (is checked once per buffer).
    fn exit(&self) -> bool { self.is_exit }

}

impl AppSoundStream {
    /// AppSoundStream constructor.
    pub fn new(recv: Option<Receiver<InteractiveEvent>>) -> AppSoundStream {
        AppSoundStream {
            chan: recv,
            is_exit: false,
            is_print: false,
            buffer: vec![]
        }
    }
}

// Main
//------------------------------

#[start]
fn start(argc: int, argv: **u8) -> int {
    // Run gui on the main thread.
    native::start(argc, argv, main)
}

fn main() {
    let mut window: GameWindowSDL2 = GameWindow::new(
        GameWindowSettings {
            title: "soundstream".to_string(),
            size: [300, 300],
            fullscreen: false,
            exit_on_esc: true,
            background_color: [0.1, 0.1, 0.1, 0.1],
        }
    );

    let mut asset_store = AssetStore::from_folder("assets");
    let mut app = App::new();
    app.run(&mut window, &mut asset_store);
}

//------------------------------
<|endoftext|>


<commit_before>use crate::{
    aws::Aws,
    cluster::Cluster,
    health::{Commit, Event, LogTail, ValidatorEvent},
    util::unix_timestamp_now,
};
use failure::{self, prelude::*};
use flate2::read::GzDecoder;
use regex::Regex;
use rusoto_kinesis::{GetRecordsInput, GetShardIteratorInput, Kinesis, Record};
use serde_json::{self, value as json};
use std::{
    collections::HashMap,
    env,
    fs::File,
    io::Write,
    sync::{
        atomic::{AtomicI64, Ordering},
        mpsc, Arc,
    },
    thread,
    time::{Duration, Instant},
};

pub struct AwsLogThread {
    aws: Aws,
    kinesis_iterator: String,
    event_sender: mpsc::Sender<ValidatorEvent>,
    startup_sender: Option<mpsc::Sender<()>>,
    event_log_file: Option<File>,
    last_seen_ts_by_validator: HashMap<String, Duration>,
    expected_validators_count: usize,
    pending_messages: Arc<AtomicI64>,

    re_commit: Regex,
    re_validator: Regex,
    re_started: Regex,
}

struct LogEntry<'a> {
    validator: String,
    message: &'a str,
    timestamp: Duration,
}

impl AwsLogThread {
    pub fn spawn_new(aws: Aws, cluster: &Cluster) -> failure::Result<LogTail> {
        let kinesis_iterator = Self::make_kinesis_iterator(&aws)?;
        let (event_sender, event_receiver) = mpsc::channel();
        // We block this method until first request to kinesis completed
        let (startup_sender, startup_receiver) = mpsc::channel();
        let event_log_file = match env::var("EVENT_LOG") {
            Ok(f) => Some(File::create(f).expect("Can't create EVENT_LOG file")),
            Err(..) => None,
        };
        let pending_messages = Arc::new(AtomicI64::new(0));
        let aws_log_thread = AwsLogThread {
            aws,
            kinesis_iterator,
            event_sender,
            startup_sender: Some(startup_sender),
            event_log_file,
            last_seen_ts_by_validator: HashMap::new(),
            expected_validators_count: cluster.instances().len(),
            pending_messages: pending_messages.clone(),

            re_commit: Regex::new(
                r"Committed.+\[id: ([a-z0-9]+), round: ([a-z0-9]+), parent_id: ([a-z0-9]+)]",
            )
            .unwrap(),
            re_validator: Regex::new(r"^validator-([0-9a-z]+)/").unwrap(),
            re_started: Regex::new(r"Chained BFT SMR started.$").unwrap(),
        };
        let builder = thread::Builder::new();
        builder
            .name("aws-log-tail".into())
            .spawn(move || aws_log_thread.run())
            .unwrap();
        startup_receiver
            .recv()
            .expect("Aws log tail thread died after first request");
        Ok(LogTail {
            event_receiver,
            pending_messages: pending_messages.clone(),
        })
    }

    fn log_start_offset_sec() -> Option<u64> {
        match env::var("LOG_START_OFFSET") {
            Ok(s) => Some(s.parse().expect("LOG_START_OFFSET env is not a number")),
            Err(..) => None,
        }
    }

    fn make_kinesis_iterator(aws: &Aws) -> failure::Result<String> {
        let stream_name = match env::var("KINESIS_STREAM") {
            Err(..) => format!("JsonEvents-{}-{}", aws.region(), aws.workplace()),
            Ok(s) => s,
        };
        let (shard_iterator_type, timestamp) = match Self::log_start_offset_sec() {
            Some(offset) => {
                let timestamp = unix_timestamp_now() - Duration::from_secs(offset);
                ("AT_TIMESTAMP", Some(timestamp.as_secs() as f64))
            }
            None => ("LATEST", None),
        };
        let response = aws
            .kc()
            .get_shard_iterator(GetShardIteratorInput {
                shard_id: "0".into(),
                shard_iterator_type: shard_iterator_type.into(),
                stream_name,
                starting_sequence_number: None,
                timestamp,
            })
            .sync()?;
        if let Some(shard_iterator) = response.shard_iterator {
            Ok(shard_iterator)
        } else {
            Err(format_err!(
                "Can not make kinesis iterator - empty response"
            ))
        }
    }

    fn run(mut self) {
        let startup_timeout_sec = match env::var("STARTUP_TIMEOUT") {
            Err(..) => 50u64,
            Ok(v) => v.parse().expect("Failed to parse STARTUP_TIMEOUT env"),
        };
        let startup_deadline = Instant::now() + Duration::from_secs(startup_timeout_sec);
        let mut kinesis_fail_retries = 0usize;
        loop {
            let response = self
                .aws
                .kc()
                .get_records(GetRecordsInput {
                    shard_iterator: self.kinesis_iterator.clone(),
                    limit: Some(10000),
                })
                .sync();
            let response = match response {
                Err(e) => {
                    println!("Kinesis failure: {:?}, retry {}", e, kinesis_fail_retries);
                    kinesis_fail_retries += 1;
                    if kinesis_fail_retries > 10 {
                        panic!("Too many kinesis failures");
                    }
                    thread::sleep(Duration::from_secs(1));
                    continue;
                }
                Ok(r) => r,
            };
            kinesis_fail_retries = 0;
            let next_iterator = response
                .next_shard_iterator
                .expect("Next iterator is expected for kinesis stream");
            let records = response.records;
            let millis_behind = response
                .millis_behind_latest
                .expect("no millis_behind_latest in kinesis response");
            self.kinesis_iterator = next_iterator;
            self.write_event_log(format!(
                "Kinesis response {} records, millis behind: {}",
                records.len(),
                millis_behind,
            ));
            for record in records {
                self.handle_kinesis_record(record)
                    .expect("Failed to process aws record");
            }
            if self.startup_sender.is_some() && millis_behind == 0 {
                if self.last_seen_ts_by_validator.len() >= self.expected_validators_count {
                    println!("Received events from all validators");
                    self.startup_sender
                        .take()
                        .unwrap()
                        .send(())
                        .expect("Startup receiver dropped");
                } else if Instant::now() > startup_deadline {
                    println!("Aws log startup deadline reached");
                    self.startup_sender
                        .take()
                        .unwrap()
                        .send(())
                        .expect("Startup receiver dropped");
                }
            }
            thread::sleep(Duration::from_millis(300))
        }
    }

    fn handle_kinesis_record(&mut self, record: Record) -> failure::Result<()> {
        let decoder = GzDecoder::new(&record.data[..]);
        let json: json::Value = serde_json::from_reader(decoder)?;
        let log_stream = json
            .get("logStream")
            .expect("No logStream in kinesis event")
            .as_str()
            .expect("logStream in kinesis event is not a string");
        let validator = self.log_stream_to_validator_short_str(log_stream);
        let validator = if let Some(validator) = validator {
            validator
        } else {
            return Ok(());
        };
        let events = json
            .get("logEvents")
            .expect("No logEvents in kinesis event");
        let events = events
            .as_array()
            .expect("logEvents in kinesis event is not array");
        self.write_event_log(format!(
            "Kinesis record for {}, {} events",
            validator,
            events.len()
        ));
        let now = unix_timestamp_now();
        let mut behind_max = 0u128;
        let mut behind_sum = 0u128;
        let mut count = 0u128;
        for event in events {
            let message = event.get("message").expect("No message in kinesis event");
            let message = message
                .as_str()
                .expect("message in kinesis event is not a string");
            let timestamp = event
                .get("timestamp")
                .expect("No timestamp in kinesis event");
            let timestamp = timestamp
                .as_u64()
                .expect("timestamp in kinesis event is not a u64");
            let timestamp = Duration::from_millis(timestamp);
            if now > timestamp {
                let behind_millis = (now - timestamp).as_millis();
                if behind_millis > behind_max {
                    behind_max = behind_millis;
                }
                behind_sum += behind_millis;
            }
            count += 1;
            let log_entry = LogEntry {
                validator: validator.clone(),
                message,
                timestamp,
            };
            self.handle_log_entry(log_entry);
        }
        if behind_max > 15_000 {
            println!(
                "Logs behind avg={} ms, max={} ms; {} entries",
                behind_sum / count,
                behind_max,
                count
            );
        }
        Ok(())
    }

    fn write_event_log(&mut self, s: String) {
        let event_log_file = if let Some(ref mut event_log_file) = self.event_log_file {
            event_log_file
        } else {
            return;
        };
        let now = unix_timestamp_now().as_millis();
        writeln!(event_log_file, "{} {}", now, s).expect("Can't write to EVENT_LOG file");
        event_log_file.flush().expect("Can't flush EVENT_LOG file");
    }

    fn handle_log_entry(&mut self, e: LogEntry) {
        let received_timestamp = unix_timestamp_now();
        let event = self.parse_log_entry(&e);
        if let Some(event) = event {
            let ve = ValidatorEvent {
                received_timestamp,
                validator: e.validator,
                timestamp: e.timestamp,
                event,
            };
            // In rare cases Kinesis delivers messages out of order
            // We are ignoring them so that invariant in CommitHistoryHealthCheck does not break
            let skip = if let Some(last) = self.last_seen_ts_by_validator.get(&ve.validator) {
                *last > ve.timestamp
            } else {
                false
            };
            let skip_msg = if skip { "; [OUT_OF_ORDER]" } else { "" };
            self.write_event_log(format!("{:?}{}", ve, skip_msg));
            if !skip {
                self.last_seen_ts_by_validator
                    .insert(ve.validator.clone(), ve.timestamp);
                self.pending_messages.fetch_add(1, Ordering::Relaxed);
                let _ignore = self.event_sender.send(ve);
            }
        }
    }

    fn parse_log_entry(&self, e: &LogEntry) -> Option<Event> {
        let commit = self.parse_commit_log_entry(&e);
        if let Some(commit) = commit {
            return Some(Event::Commit(commit));
        }
        let consensus_started = self.parse_consensus_started_log_entry(&e);
        if consensus_started.is_some() {
            return Some(Event::ConsensusStarted);
        }
        None
    }

    fn log_stream_to_validator_short_str(&self, s: &str) -> Option<String> {
        let cap = self.re_validator.captures(s);
        if let Some(cap) = cap {
            Some(cap[1].into())
        } else {
            None
        }
    }

    fn parse_commit_log_entry(&self, e: &LogEntry) -> Option<Commit> {
        let cap = match self.re_commit.captures(e.message) {
            Some(cap) => cap,
            None => return None,
        };
        let commit = &cap[1];
        let round = &cap[2];
        let parent = &cap[3];
        let round = match round.parse::<u64>() {
            Ok(round) => round,
            Err(..) => return None,
        };
        Some(Commit {
            commit: commit.into(),
            round,
            parent: parent.into(),
        })
    }

    fn parse_consensus_started_log_entry(&self, e: &LogEntry) -> Option<()> {
        if self.re_started.is_match(e.message) {
            Some(())
        } else {
            None
        }
    }
}
<commit_msg>Rewrite an example of `option.map()`<commit_after>use crate::{
    aws::Aws,
    cluster::Cluster,
    health::{Commit, Event, LogTail, ValidatorEvent},
    util::unix_timestamp_now,
};
use failure::{self, prelude::*};
use flate2::read::GzDecoder;
use regex::Regex;
use rusoto_kinesis::{GetRecordsInput, GetShardIteratorInput, Kinesis, Record};
use serde_json::{self, value as json};
use std::{
    collections::HashMap,
    env,
    fs::File,
    io::Write,
    sync::{
        atomic::{AtomicI64, Ordering},
        mpsc, Arc,
    },
    thread,
    time::{Duration, Instant},
};

pub struct AwsLogThread {
    aws: Aws,
    kinesis_iterator: String,
    event_sender: mpsc::Sender<ValidatorEvent>,
    startup_sender: Option<mpsc::Sender<()>>,
    event_log_file: Option<File>,
    last_seen_ts_by_validator: HashMap<String, Duration>,
    expected_validators_count: usize,
    pending_messages: Arc<AtomicI64>,

    re_commit: Regex,
    re_validator: Regex,
    re_started: Regex,
}

struct LogEntry<'a> {
    validator: String,
    message: &'a str,
    timestamp: Duration,
}

impl AwsLogThread {
    pub fn spawn_new(aws: Aws, cluster: &Cluster) -> failure::Result<LogTail> {
        let kinesis_iterator = Self::make_kinesis_iterator(&aws)?;
        let (event_sender, event_receiver) = mpsc::channel();
        // We block this method until first request to kinesis completed
        let (startup_sender, startup_receiver) = mpsc::channel();
        let event_log_file = match env::var("EVENT_LOG") {
            Ok(f) => Some(File::create(f).expect("Can't create EVENT_LOG file")),
            Err(..) => None,
        };
        let pending_messages = Arc::new(AtomicI64::new(0));
        let aws_log_thread = AwsLogThread {
            aws,
            kinesis_iterator,
            event_sender,
            startup_sender: Some(startup_sender),
            event_log_file,
            last_seen_ts_by_validator: HashMap::new(),
            expected_validators_count: cluster.instances().len(),
            pending_messages: pending_messages.clone(),

            re_commit: Regex::new(
                r"Committed.+\[id: ([a-z0-9]+), round: ([a-z0-9]+), parent_id: ([a-z0-9]+)]",
            )
            .unwrap(),
            re_validator: Regex::new(r"^validator-([0-9a-z]+)/").unwrap(),
            re_started: Regex::new(r"Chained BFT SMR started.$").unwrap(),
        };
        let builder = thread::Builder::new();
        builder
            .name("aws-log-tail".into())
            .spawn(move || aws_log_thread.run())
            .unwrap();
        startup_receiver
            .recv()
            .expect("Aws log tail thread died after first request");
        Ok(LogTail {
            event_receiver,
            pending_messages: pending_messages.clone(),
        })
    }

    fn log_start_offset_sec() -> Option<u64> {
        match env::var("LOG_START_OFFSET") {
            Ok(s) => Some(s.parse().expect("LOG_START_OFFSET env is not a number")),
            Err(..) => None,
        }
    }

    fn make_kinesis_iterator(aws: &Aws) -> failure::Result<String> {
        let stream_name = match env::var("KINESIS_STREAM") {
            Err(..) => format!("JsonEvents-{}-{}", aws.region(), aws.workplace()),
            Ok(s) => s,
        };
        let (shard_iterator_type, timestamp) = match Self::log_start_offset_sec() {
            Some(offset) => {
                let timestamp = unix_timestamp_now() - Duration::from_secs(offset);
                ("AT_TIMESTAMP", Some(timestamp.as_secs() as f64))
            }
            None => ("LATEST", None),
        };
        let response = aws
            .kc()
            .get_shard_iterator(GetShardIteratorInput {
                shard_id: "0".into(),
                shard_iterator_type: shard_iterator_type.into(),
                stream_name,
                starting_sequence_number: None,
                timestamp,
            })
            .sync()?;
        if let Some(shard_iterator) = response.shard_iterator {
            Ok(shard_iterator)
        } else {
            Err(format_err!(
                "Can not make kinesis iterator - empty response"
            ))
        }
    }

    fn run(mut self) {
        let startup_timeout_sec = match env::var("STARTUP_TIMEOUT") {
            Err(..) => 50u64,
            Ok(v) => v.parse().expect("Failed to parse STARTUP_TIMEOUT env"),
        };
        let startup_deadline = Instant::now() + Duration::from_secs(startup_timeout_sec);
        let mut kinesis_fail_retries = 0usize;
        loop {
            let response = self
                .aws
                .kc()
                .get_records(GetRecordsInput {
                    shard_iterator: self.kinesis_iterator.clone(),
                    limit: Some(10000),
                })
                .sync();
            let response = match response {
                Err(e) => {
                    println!("Kinesis failure: {:?}, retry {}", e, kinesis_fail_retries);
                    kinesis_fail_retries += 1;
                    if kinesis_fail_retries > 10 {
                        panic!("Too many kinesis failures");
                    }
                    thread::sleep(Duration::from_secs(1));
                    continue;
                }
                Ok(r) => r,
            };
            kinesis_fail_retries = 0;
            let next_iterator = response
                .next_shard_iterator
                .expect("Next iterator is expected for kinesis stream");
            let records = response.records;
            let millis_behind = response
                .millis_behind_latest
                .expect("no millis_behind_latest in kinesis response");
            self.kinesis_iterator = next_iterator;
            self.write_event_log(format!(
                "Kinesis response {} records, millis behind: {}",
                records.len(),
                millis_behind,
            ));
            for record in records {
                self.handle_kinesis_record(record)
                    .expect("Failed to process aws record");
            }
            if self.startup_sender.is_some() && millis_behind == 0 {
                if self.last_seen_ts_by_validator.len() >= self.expected_validators_count {
                    println!("Received events from all validators");
                    self.startup_sender
                        .take()
                        .unwrap()
                        .send(())
                        .expect("Startup receiver dropped");
                } else if Instant::now() > startup_deadline {
                    println!("Aws log startup deadline reached");
                    self.startup_sender
                        .take()
                        .unwrap()
                        .send(())
                        .expect("Startup receiver dropped");
                }
            }
            thread::sleep(Duration::from_millis(300))
        }
    }

    fn handle_kinesis_record(&mut self, record: Record) -> failure::Result<()> {
        let decoder = GzDecoder::new(&record.data[..]);
        let json: json::Value = serde_json::from_reader(decoder)?;
        let log_stream = json
            .get("logStream")
            .expect("No logStream in kinesis event")
            .as_str()
            .expect("logStream in kinesis event is not a string");
        let validator = self.log_stream_to_validator_short_str(log_stream);
        let validator = if let Some(validator) = validator {
            validator
        } else {
            return Ok(());
        };
        let events = json
            .get("logEvents")
            .expect("No logEvents in kinesis event");
        let events = events
            .as_array()
            .expect("logEvents in kinesis event is not array");
        self.write_event_log(format!(
            "Kinesis record for {}, {} events",
            validator,
            events.len()
        ));
        let now = unix_timestamp_now();
        let mut behind_max = 0u128;
        let mut behind_sum = 0u128;
        let mut count = 0u128;
        for event in events {
            let message = event.get("message").expect("No message in kinesis event");
            let message = message
                .as_str()
                .expect("message in kinesis event is not a string");
            let timestamp = event
                .get("timestamp")
                .expect("No timestamp in kinesis event");
            let timestamp = timestamp
                .as_u64()
                .expect("timestamp in kinesis event is not a u64");
            let timestamp = Duration::from_millis(timestamp);
            if now > timestamp {
                let behind_millis = (now - timestamp).as_millis();
                if behind_millis > behind_max {
                    behind_max = behind_millis;
                }
                behind_sum += behind_millis;
            }
            count += 1;
            let log_entry = LogEntry {
                validator: validator.clone(),
                message,
                timestamp,
            };
            self.handle_log_entry(log_entry);
        }
        if behind_max > 15_000 {
            println!(
                "Logs behind avg={} ms, max={} ms; {} entries",
                behind_sum / count,
                behind_max,
                count
            );
        }
        Ok(())
    }

    fn write_event_log(&mut self, s: String) {
        let event_log_file = if let Some(ref mut event_log_file) = self.event_log_file {
            event_log_file
        } else {
            return;
        };
        let now = unix_timestamp_now().as_millis();
        writeln!(event_log_file, "{} {}", now, s).expect("Can't write to EVENT_LOG file");
        event_log_file.flush().expect("Can't flush EVENT_LOG file");
    }

    fn handle_log_entry(&mut self, e: LogEntry) {
        let received_timestamp = unix_timestamp_now();
        let event = self.parse_log_entry(&e);
        if let Some(event) = event {
            let ve = ValidatorEvent {
                received_timestamp,
                validator: e.validator,
                timestamp: e.timestamp,
                event,
            };
            // In rare cases Kinesis delivers messages out of order
            // We are ignoring them so that invariant in CommitHistoryHealthCheck does not break
            let skip = if let Some(last) = self.last_seen_ts_by_validator.get(&ve.validator) {
                *last > ve.timestamp
            } else {
                false
            };
            let skip_msg = if skip { "; [OUT_OF_ORDER]" } else { "" };
            self.write_event_log(format!("{:?}{}", ve, skip_msg));
            if !skip {
                self.last_seen_ts_by_validator
                    .insert(ve.validator.clone(), ve.timestamp);
                self.pending_messages.fetch_add(1, Ordering::Relaxed);
                let _ignore = self.event_sender.send(ve);
            }
        }
    }

    fn parse_log_entry(&self, e: &LogEntry) -> Option<Event> {
        let commit = self.parse_commit_log_entry(&e);
        if let Some(commit) = commit {
            return Some(Event::Commit(commit));
        }
        let consensus_started = self.parse_consensus_started_log_entry(&e);
        if consensus_started.is_some() {
            return Some(Event::ConsensusStarted);
        }
        None
    }

    fn log_stream_to_validator_short_str(&self, s: &str) -> Option<String> {
        let cap = self.re_validator.captures(s);
        cap.map(|cap| cap[1].into())
    }

    fn parse_commit_log_entry(&self, e: &LogEntry) -> Option<Commit> {
        let cap = match self.re_commit.captures(e.message) {
            Some(cap) => cap,
            None => return None,
        };
        let commit = &cap[1];
        let round = &cap[2];
        let parent = &cap[3];
        let round = match round.parse::<u64>() {
            Ok(round) => round,
            Err(..) => return None,
        };
        Some(Commit {
            commit: commit.into(),
            round,
            parent: parent.into(),
        })
    }

    fn parse_consensus_started_log_entry(&self, e: &LogEntry) -> Option<()> {
        if self.re_started.is_match(e.message) {
            Some(())
        } else {
            None
        }
    }
}
<|endoftext|>


<commit_before>#![feature(plugin)]
#![plugin(serde2_macros)]

extern crate serde2;

use std::collections::BTreeMap;
use serde2::json::{self, Value};

macro_rules! btreemap {
    () => {
        BTreeMap::new()
    };
    ($($key:expr => $value:expr),+) => {
        {
            let mut map = BTreeMap::new();
            $(map.insert($key, $value);)+
            map
        }
    }
}

/*
trait Trait {
    type Type;
}
*/

#[derive(Debug, PartialEq)]
#[derive_serialize]
struct NamedUnit;

#[derive(Debug, PartialEq)]
#[derive_serialize]
struct NamedTuple<'a, 'b, A: 'a, B: 'b, C>(&'a A, &'b mut B, C);

#[derive(Debug, PartialEq)]
#[derive_serialize]
struct NamedMap<'a, 'b, A: 'a, B: 'b, C> {
    a: &'a A,
    b: &'b mut B,
    c: C,
}

#[derive(Debug, PartialEq)]
#[derive_serialize]
//#[derive_deserialize]
enum Enum<'a, B: 'a, C: /* Trait + */ 'a, D> where D: /* Trait + */ 'a {
    Unit,
    Seq(
        i8,
        B,
        &'a C,
        //B::Type,
        &'a mut D,
        //<D as Trait>::Type,
    ),
    Map {
        a: i8,
        b: B,
        c: &'a C,
        //d: C::Type,
        e: &'a mut D,
        //f: <D as Trait>::Type,
    },
}

#[test]
fn test_named_unit() {
    let named_unit = NamedUnit;

    assert_eq!(
        json::to_string(&named_unit).unwrap(),
        "null".to_string()
    );

    assert_eq!(
        json::to_value(&named_unit),
        Value::Null
    );

    /*
    let v = json::from_str("null").unwrap();
    assert_eq!(v, named_unit);

    let v = json::from_value(Value::Null).unwrap();
    assert_eq!(v, named_unit);
    */
}

#[test]
fn test_named_tuple() {
    let a = 5;
    let mut b = 6;
    let c = 7;
    let named_tuple = NamedTuple(&a, &mut b, c);

    assert_eq!(
        json::to_string(&named_tuple).unwrap(),
        "[5,6,7]"
    );

    assert_eq!(
        json::to_value(&named_tuple),
        Value::Array(vec![Value::I64(5), Value::I64(6), Value::I64(7)])
    );
}

#[test]
fn test_named_map() {
    let a = 5;
    let mut b = 6;
    let c = 7;
    let named_map = NamedMap {
        a: &a,
        b: &mut b,
        c: c,
    };

    assert_eq!(
        json::to_string(&named_map).unwrap(),
        "{\"a\":5,\"b\":6,\"c\":7}"
    );

    assert_eq!(
        json::to_value(&named_map),
        Value::Object(btreemap![
            "a".to_string() => Value::I64(5),
            "b".to_string() => Value::I64(6),
            "c".to_string() => Value::I64(7)
        ])
    );
}

#[test]
fn test_enum_unit() {
    assert_eq!(
        json::to_string(&Enum::Unit::<u32, u32, u32>).unwrap(),
        "{\"Unit\":[]}"
    );

    assert_eq!(
        json::to_value(&Enum::Unit::<u32, u32, u32>),
        Value::Object(btreemap!(
            "Unit".to_string() => Value::Array(vec![]))
        )
    );
}

#[test]
fn test_enum_seq() {
    let a = 1;
    let b = 2;
    let c = 3;
    //let d = 4;
    let mut e = 5;
    //let f = 6;

    assert_eq!(
        json::to_string(&Enum::Seq(
            a,
            b,
            &c,
            //d,
            &mut e,
            //f,
        )).unwrap(),
        "{\"Seq\":[1,2,3,5]}".to_string()
    );

    assert_eq!(
        json::to_value(&Enum::Seq(
            a,
            b,
            &c,
            //d,
            &mut e,
            //e,
        )),
        Value::Object(btreemap!(
            "Seq".to_string() => Value::Array(vec![
                Value::I64(1),
                Value::I64(2),
                Value::I64(3),
                //Value::I64(4),
                Value::I64(5),
                //Value::I64(6),
            ])
        ))
    );
}

#[test]
fn test_enum_map() {
    let a = 1;
    let b = 2;
    let c = 3;
    //let d = 4;
    let mut e = 5;
    //let f = 6;

    assert_eq!(
        json::to_string(&Enum::Map {
            a: a,
            b: b,
            c: &c,
            //d: d,
            e: &mut e,
            //f: f,
        }).unwrap(),
        "{\"Map\":{\"a\":1,\"b\":2,\"c\":3,\"e\":5}}".to_string()
    );

    assert_eq!(
        json::to_value(&Enum::Map {
            a: a,
            b: b,
            c: &c,
            //d: d,
            e: &mut e,
            //f: f,
        }),
        Value::Object(btreemap!(
            "Map".to_string() => Value::Object(btreemap![
                "a".to_string() => Value::I64(1),
                "b".to_string() => Value::I64(2),
                "c".to_string() => Value::I64(3),
                //"d".to_string() => Value::I64(4)
                "e".to_string() => Value::I64(5)
                //"f".to_string() => Value::I64(6)
            ])
        ))
    );
}
<commit_msg>Add test for deserializing named units<commit_after>#![feature(plugin)]
#![plugin(serde2_macros)]

extern crate serde2;

use std::collections::BTreeMap;
use serde2::json::{self, Value};

macro_rules! btreemap {
    () => {
        BTreeMap::new()
    };
    ($($key:expr => $value:expr),+) => {
        {
            let mut map = BTreeMap::new();
            $(map.insert($key, $value);)+
            map
        }
    }
}

/*
trait Trait {
    type Type;
}
*/

#[derive(Debug, PartialEq)]
#[derive_serialize]
#[derive_deserialize]
struct NamedUnit;

#[derive(Debug, PartialEq)]
#[derive_serialize]
struct NamedTuple<'a, 'b, A: 'a, B: 'b, C>(&'a A, &'b mut B, C);

#[derive(Debug, PartialEq)]
#[derive_serialize]
struct NamedMap<'a, 'b, A: 'a, B: 'b, C> {
    a: &'a A,
    b: &'b mut B,
    c: C,
}

#[derive(Debug, PartialEq)]
#[derive_serialize]
//#[derive_deserialize]
enum Enum<'a, B: 'a, C: /* Trait + */ 'a, D> where D: /* Trait + */ 'a {
    Unit,
    Seq(
        i8,
        B,
        &'a C,
        //B::Type,
        &'a mut D,
        //<D as Trait>::Type,
    ),
    Map {
        a: i8,
        b: B,
        c: &'a C,
        //d: C::Type,
        e: &'a mut D,
        //f: <D as Trait>::Type,
    },
}

#[test]
fn test_named_unit() {
    let named_unit = NamedUnit;

    assert_eq!(
        json::to_string(&named_unit).unwrap(),
        "null".to_string()
    );

    assert_eq!(
        json::to_value(&named_unit),
        Value::Null
    );

    let v = json::from_str("null").unwrap();
    assert_eq!(v, named_unit);

    let v = json::from_value(Value::Null).unwrap();
    assert_eq!(v, named_unit);
}

#[test]
fn test_named_tuple() {
    let a = 5;
    let mut b = 6;
    let c = 7;
    let named_tuple = NamedTuple(&a, &mut b, c);

    assert_eq!(
        json::to_string(&named_tuple).unwrap(),
        "[5,6,7]"
    );

    assert_eq!(
        json::to_value(&named_tuple),
        Value::Array(vec![Value::I64(5), Value::I64(6), Value::I64(7)])
    );
}

#[test]
fn test_named_map() {
    let a = 5;
    let mut b = 6;
    let c = 7;
    let named_map = NamedMap {
        a: &a,
        b: &mut b,
        c: c,
    };

    assert_eq!(
        json::to_string(&named_map).unwrap(),
        "{\"a\":5,\"b\":6,\"c\":7}"
    );

    assert_eq!(
        json::to_value(&named_map),
        Value::Object(btreemap![
            "a".to_string() => Value::I64(5),
            "b".to_string() => Value::I64(6),
            "c".to_string() => Value::I64(7)
        ])
    );
}

#[test]
fn test_enum_unit() {
    assert_eq!(
        json::to_string(&Enum::Unit::<u32, u32, u32>).unwrap(),
        "{\"Unit\":[]}"
    );

    assert_eq!(
        json::to_value(&Enum::Unit::<u32, u32, u32>),
        Value::Object(btreemap!(
            "Unit".to_string() => Value::Array(vec![]))
        )
    );
}

#[test]
fn test_enum_seq() {
    let a = 1;
    let b = 2;
    let c = 3;
    //let d = 4;
    let mut e = 5;
    //let f = 6;

    assert_eq!(
        json::to_string(&Enum::Seq(
            a,
            b,
            &c,
            //d,
            &mut e,
            //f,
        )).unwrap(),
        "{\"Seq\":[1,2,3,5]}".to_string()
    );

    assert_eq!(
        json::to_value(&Enum::Seq(
            a,
            b,
            &c,
            //d,
            &mut e,
            //e,
        )),
        Value::Object(btreemap!(
            "Seq".to_string() => Value::Array(vec![
                Value::I64(1),
                Value::I64(2),
                Value::I64(3),
                //Value::I64(4),
                Value::I64(5),
                //Value::I64(6),
            ])
        ))
    );
}

#[test]
fn test_enum_map() {
    let a = 1;
    let b = 2;
    let c = 3;
    //let d = 4;
    let mut e = 5;
    //let f = 6;

    assert_eq!(
        json::to_string(&Enum::Map {
            a: a,
            b: b,
            c: &c,
            //d: d,
            e: &mut e,
            //f: f,
        }).unwrap(),
        "{\"Map\":{\"a\":1,\"b\":2,\"c\":3,\"e\":5}}".to_string()
    );

    assert_eq!(
        json::to_value(&Enum::Map {
            a: a,
            b: b,
            c: &c,
            //d: d,
            e: &mut e,
            //f: f,
        }),
        Value::Object(btreemap!(
            "Map".to_string() => Value::Object(btreemap![
                "a".to_string() => Value::I64(1),
                "b".to_string() => Value::I64(2),
                "c".to_string() => Value::I64(3),
                //"d".to_string() => Value::I64(4)
                "e".to_string() => Value::I64(5)
                //"f".to_string() => Value::I64(6)
            ])
        ))
    );
}
<|endoftext|>


<commit_before>#![feature(plugin)]
#![plugin(serde2_macros)]

extern crate serde2;

trait Trait {
    type Type;
}

#[derive_serialize]
//#[derive_deserialize]
enum Enum<'a, A: 'a, B: /* Trait + */ 'a, C> where C: /* Trait + */ 'a {
    Unit,
    Seq(
        i8,
        &'a A,
        &'a B,
        //B::Type,
        &'a C,
        //<C as Trait>::Type,
    ),
    Map {
        a: i8,
        b: &'a A,
        c: &'a B,
        //d: B::Type,
        e: &'a C,
        //f: <C as Trait>::Type,
    },
}

fn main() {}
<commit_msg>Silence a warning<commit_after>#![feature(plugin)]
#![plugin(serde2_macros)]

extern crate serde2;

/*
trait Trait {
    type Type;
}
*/

#[derive_serialize]
//#[derive_deserialize]
enum Enum<'a, A: 'a, B: /* Trait + */ 'a, C> where C: /* Trait + */ 'a {
    Unit,
    Seq(
        i8,
        &'a A,
        &'a B,
        //B::Type,
        &'a C,
        //<C as Trait>::Type,
    ),
    Map {
        a: i8,
        b: &'a A,
        c: &'a B,
        //d: B::Type,
        e: &'a C,
        //f: <C as Trait>::Type,
    },
}

fn main() {}
<|endoftext|>