<commit_before>mod ringbuffer;

use anyhow::Result;
use crossterm::event::{KeyEvent, KeyModifiers};
use crossterm::{
    event::{self, DisableMouseCapture, EnableMouseCapture, Event as CEvent, KeyCode},
    execute,
    terminal::{disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen},
};
use histogram::Histogram;
use pinger::{ping, PingResult};
use std::io;
use std::io::Write;
use std::iter;
use std::ops::Add;
use std::sync::atomic::Ordering;
use std::sync::mpsc;
use std::thread;
use std::time::Duration;
use structopt::StructOpt;
use tui::backend::CrosstermBackend;
use tui::layout::{Constraint, Direction, Layout};
use tui::style::{Color, Style};
use tui::text::Span;
use tui::widgets::{Axis, Block, Borders, Chart, Dataset, GraphType, Paragraph};
use tui::{symbols, Terminal};

#[derive(Debug, StructOpt)]
#[structopt(name = "gping", about = "Ping, but with a graph.")]
struct Args {
    #[structopt(help = "Hosts or IPs to ping", required = true)]
    hosts: Vec<String>,
    #[structopt(
        short,
        long,
        default_value = "100",
        help = "Determines the number pings to display."
    )]
    buffer: usize,
}

struct App {
    styles: Vec<Style>,
    data: Vec<ringbuffer::FixedRingBuffer<(f64, f64)>>,
    capacity: usize,
    idx: Vec<i64>,
    window_min: Vec<f64>,
    window_max: Vec<f64>,
}

impl App {
    fn new(host_count: usize, capacity: usize) -> Self {
        App {
            styles: (0..host_count)
                .map(|i| Style::default().fg(Color::Indexed(i as u8 + 1)))
                .collect(),
            data: (0..host_count)
                .map(|_| ringbuffer::FixedRingBuffer::new(capacity))
                .collect(),
            capacity,
            idx: vec![0; host_count],
            window_min: vec![0.0; host_count],
            window_max: vec![capacity as f64; host_count],
        }
    }
    fn update(&mut self, host_id: usize, item: Option<Duration>) {
        self.idx[host_id] += 1;
        let data = &mut self.data[host_id];
        if data.len() >= self.capacity {
            self.window_min[host_id] += 1_f64;
            self.window_max[host_id] += 1_f64;
        }
        match item {
            Some(dur) => data.push((self.idx[host_id] as f64, dur.as_micros() as f64)),
            None => data.push((self.idx[host_id] as f64, 0_f64)),
        }
    }
    fn stats(&self) -> Vec<Histogram> {
        self.data
            .iter()
            .map(|data| {
                let mut hist = Histogram::new();

                for (_, val) in data.iter().filter(|v| v.1 != 0f64) {
                    hist.increment(*val as u64).unwrap_or(());
                }

                hist
            })
            .collect()
    }
    fn x_axis_bounds(&self) -> [f64; 2] {
        [
            self.window_min.iter().fold(f64::INFINITY, |a, &b| a.min(b)),
            self.window_max.iter().fold(0f64, |a, &b| a.max(b)),
        ]
    }
    fn y_axis_bounds(&self) -> [f64; 2] {
        let iter = self
            .data
            .iter()
            .map(|b| b.as_slice())
            .flatten()
            .map(|v| v.1);
        let min = iter.clone().fold(f64::INFINITY, |a, b| a.min(b));
        let max = iter.fold(0f64, |a, b| a.max(b));
        // Add a 10% buffer to the top and bottom
        let max_10_percent = (max * 10_f64) / 100_f64;
        let min_10_percent = (min * 10_f64) / 100_f64;
        [min - min_10_percent, max + max_10_percent]
    }
    fn y_axis_labels(&self, bounds: [f64; 2]) -> Vec<Span> {
        // Split into 5 sections
        let min = bounds[0];
        let max = bounds[1];

        let difference = max - min;
        let increment = Duration::from_micros((difference / 3f64) as u64);
        let duration = Duration::from_micros(min as u64);

        (0..7)
            .map(|i| Span::raw(format!("{:?}", duration.add(increment * i))))
            .collect()
    }
}

#[derive(Debug)]
enum Event {
    Update(usize, PingResult),
    Input(KeyEvent),
}

fn main() -> Result<()> {
    let args = Args::from_args();
    let mut app = App::new(args.hosts.len(), args.buffer);
    enable_raw_mode()?;
    let mut stdout = io::stdout();
    execute!(stdout, EnterAlternateScreen, EnableMouseCapture)?;
    let backend = CrosstermBackend::new(stdout);

    let mut terminal = Terminal::new(backend)?;

    terminal.clear()?;

    let (key_tx, rx) = mpsc::channel();

    let mut threads = vec![];

    let killed = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));

    for (host_id, host) in args.hosts.iter().cloned().enumerate() {
        let ping_tx = key_tx.clone();

        let killed_ping = std::sync::Arc::clone(&killed);
        // Pump ping messages into the queue
        let ping_thread = thread::spawn(move || -> Result<()> {
            let stream = ping(host)?;
            while !killed_ping.load(Ordering::Acquire) {
                ping_tx.send(Event::Update(host_id, stream.recv()?))?;
            }
            Ok(())
        });
        threads.push(ping_thread);
    }

    // Pump keyboard messages into the queue
    let killed_thread = std::sync::Arc::clone(&killed);
    let key_thread = thread::spawn(move || -> Result<()> {
        while !killed_thread.load(Ordering::Acquire) {
            if event::poll(Duration::from_secs(1))? {
                if let CEvent::Key(key) = event::read()? {
                    key_tx.send(Event::Input(key))?;
                }
            }
        }
        Ok(())
    });
    threads.push(key_thread);

    loop {
        match rx.recv()? {
            Event::Update(host_id, ping_result) => {
                match ping_result {
                    PingResult::Pong(duration) => app.update(host_id, Some(duration)),
                    PingResult::Timeout => app.update(host_id, None),
                };
                terminal.draw(|f| {
                    let chunks = Layout::default()
                        .direction(Direction::Vertical)
                        .margin(2)
                        .constraints(
                            iter::repeat(Constraint::Length(1))
                                .take(args.hosts.len())
                                .chain(iter::once(Constraint::Percentage(10)))
                                .collect::<Vec<_>>()
                                .as_ref(),
                        )
                        .split(f.size());
                    for (((host_id, host), stats), &style) in args
                        .hosts
                        .iter()
                        .enumerate()
                        .zip(app.stats())
                        .zip(&app.styles)
                    {
                        let header_layout = Layout::default()
                            .direction(Direction::Horizontal)
                            .constraints(
                                [
                                    Constraint::Percentage(25),
                                    Constraint::Percentage(25),
                                    Constraint::Percentage(25),
                                    Constraint::Percentage(25),
                                ]
                                .as_ref(),
                            )
                            .split(chunks[host_id]);

                        f.render_widget(
                            Paragraph::new(format!("Pinging {}", host)).style(style),
                            header_layout[0],
                        );

                        f.render_widget(
                            Paragraph::new(format!(
                                "min {:?}",
                                Duration::from_micros(stats.minimum().unwrap_or(0))
                            ))
                            .style(style),
                            header_layout[1],
                        );
                        f.render_widget(
                            Paragraph::new(format!(
                                "max {:?}",
                                Duration::from_micros(stats.maximum().unwrap_or(0))
                            ))
                            .style(style),
                            header_layout[2],
                        );
                        f.render_widget(
                            Paragraph::new(format!(
                                "p95 {:?}",
                                Duration::from_micros(stats.percentile(95.0).unwrap_or(0))
                            ))
                            .style(style),
                            header_layout[3],
                        );
                    }

                    let datasets: Vec<_> = app
                        .data
                        .iter()
                        .zip(&app.styles)
                        .map(|(data, &style)| {
                            Dataset::default()
                                .marker(symbols::Marker::Braille)
                                .style(style)
                                .graph_type(GraphType::Line)
                                .data(data.as_slice())
                        })
                        .collect();

                    let y_axis_bounds = app.y_axis_bounds();

                    let chart = Chart::new(datasets)
                        .block(Block::default().borders(Borders::NONE))
                        .x_axis(
                            Axis::default()
                                .style(Style::default().fg(Color::Gray))
                                .bounds(app.x_axis_bounds()),
                        )
                        .y_axis(
                            Axis::default()
                                .style(Style::default().fg(Color::Gray))
                                .bounds(y_axis_bounds)
                                .labels(app.y_axis_labels(y_axis_bounds)),
                        );
                    f.render_widget(chart, chunks[args.hosts.len()]);
                })?;
            }
            Event::Input(input) => match input.code {
                KeyCode::Char('q') | KeyCode::Esc => {
                    killed.store(true, Ordering::Release);
                    break;
                }
                KeyCode::Char('c') if input.modifiers == KeyModifiers::CONTROL => {
                    killed.store(true, Ordering::Release);
                    break;
                }
                _ => {}
            },
        }
    }

    for thread in threads {
        thread.join().unwrap()?;
    }

    disable_raw_mode()?;
    execute!(
        terminal.backend_mut(),
        LeaveAlternateScreen,
        DisableMouseCapture
    )?;
    terminal.show_cursor()?;

    Ok(())
}
<commit_msg>Reduce keypress polling to 100ms from 1 second. Closes #95<commit_after>mod ringbuffer;

use anyhow::Result;
use crossterm::event::{KeyEvent, KeyModifiers};
use crossterm::{
    event::{self, DisableMouseCapture, EnableMouseCapture, Event as CEvent, KeyCode},
    execute,
    terminal::{disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen},
};
use histogram::Histogram;
use pinger::{ping, PingResult};
use std::io;
use std::io::Write;
use std::iter;
use std::ops::Add;
use std::sync::atomic::Ordering;
use std::sync::mpsc;
use std::thread;
use std::time::Duration;
use structopt::StructOpt;
use tui::backend::CrosstermBackend;
use tui::layout::{Constraint, Direction, Layout};
use tui::style::{Color, Style};
use tui::text::Span;
use tui::widgets::{Axis, Block, Borders, Chart, Dataset, GraphType, Paragraph};
use tui::{symbols, Terminal};

#[derive(Debug, StructOpt)]
#[structopt(name = "gping", about = "Ping, but with a graph.")]
struct Args {
    #[structopt(help = "Hosts or IPs to ping", required = true)]
    hosts: Vec<String>,
    #[structopt(
        short,
        long,
        default_value = "100",
        help = "Determines the number pings to display."
    )]
    buffer: usize,
}

struct App {
    styles: Vec<Style>,
    data: Vec<ringbuffer::FixedRingBuffer<(f64, f64)>>,
    capacity: usize,
    idx: Vec<i64>,
    window_min: Vec<f64>,
    window_max: Vec<f64>,
}

impl App {
    fn new(host_count: usize, capacity: usize) -> Self {
        App {
            styles: (0..host_count)
                .map(|i| Style::default().fg(Color::Indexed(i as u8 + 1)))
                .collect(),
            data: (0..host_count)
                .map(|_| ringbuffer::FixedRingBuffer::new(capacity))
                .collect(),
            capacity,
            idx: vec![0; host_count],
            window_min: vec![0.0; host_count],
            window_max: vec![capacity as f64; host_count],
        }
    }
    fn update(&mut self, host_id: usize, item: Option<Duration>) {
        self.idx[host_id] += 1;
        let data = &mut self.data[host_id];
        if data.len() >= self.capacity {
            self.window_min[host_id] += 1_f64;
            self.window_max[host_id] += 1_f64;
        }
        match item {
            Some(dur) => data.push((self.idx[host_id] as f64, dur.as_micros() as f64)),
            None => data.push((self.idx[host_id] as f64, 0_f64)),
        }
    }
    fn stats(&self) -> Vec<Histogram> {
        self.data
            .iter()
            .map(|data| {
                let mut hist = Histogram::new();

                for (_, val) in data.iter().filter(|v| v.1 != 0f64) {
                    hist.increment(*val as u64).unwrap_or(());
                }

                hist
            })
            .collect()
    }
    fn x_axis_bounds(&self) -> [f64; 2] {
        [
            self.window_min.iter().fold(f64::INFINITY, |a, &b| a.min(b)),
            self.window_max.iter().fold(0f64, |a, &b| a.max(b)),
        ]
    }
    fn y_axis_bounds(&self) -> [f64; 2] {
        let iter = self
            .data
            .iter()
            .map(|b| b.as_slice())
            .flatten()
            .map(|v| v.1);
        let min = iter.clone().fold(f64::INFINITY, |a, b| a.min(b));
        let max = iter.fold(0f64, |a, b| a.max(b));
        // Add a 10% buffer to the top and bottom
        let max_10_percent = (max * 10_f64) / 100_f64;
        let min_10_percent = (min * 10_f64) / 100_f64;
        [min - min_10_percent, max + max_10_percent]
    }
    fn y_axis_labels(&self, bounds: [f64; 2]) -> Vec<Span> {
        // Split into 5 sections
        let min = bounds[0];
        let max = bounds[1];

        let difference = max - min;
        let increment = Duration::from_micros((difference / 3f64) as u64);
        let duration = Duration::from_micros(min as u64);

        (0..7)
            .map(|i| Span::raw(format!("{:?}", duration.add(increment * i))))
            .collect()
    }
}

#[derive(Debug)]
enum Event {
    Update(usize, PingResult),
    Input(KeyEvent),
}

fn main() -> Result<()> {
    let args = Args::from_args();
    let mut app = App::new(args.hosts.len(), args.buffer);
    enable_raw_mode()?;
    let mut stdout = io::stdout();
    execute!(stdout, EnterAlternateScreen, EnableMouseCapture)?;
    let backend = CrosstermBackend::new(stdout);

    let mut terminal = Terminal::new(backend)?;

    terminal.clear()?;

    let (key_tx, rx) = mpsc::channel();

    let mut threads = vec![];

    let killed = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));

    for (host_id, host) in args.hosts.iter().cloned().enumerate() {
        let ping_tx = key_tx.clone();

        let killed_ping = std::sync::Arc::clone(&killed);
        // Pump ping messages into the queue
        let ping_thread = thread::spawn(move || -> Result<()> {
            let stream = ping(host)?;
            while !killed_ping.load(Ordering::Acquire) {
                ping_tx.send(Event::Update(host_id, stream.recv()?))?;
            }
            Ok(())
        });
        threads.push(ping_thread);
    }

    // Pump keyboard messages into the queue
    let killed_thread = std::sync::Arc::clone(&killed);
    let key_thread = thread::spawn(move || -> Result<()> {
        while !killed_thread.load(Ordering::Acquire) {
            if event::poll(Duration::from_millis(100))? {
                if let CEvent::Key(key) = event::read()? {
                    key_tx.send(Event::Input(key))?;
                }
            }
        }
        Ok(())
    });
    threads.push(key_thread);

    loop {
        match rx.recv()? {
            Event::Update(host_id, ping_result) => {
                match ping_result {
                    PingResult::Pong(duration) => app.update(host_id, Some(duration)),
                    PingResult::Timeout => app.update(host_id, None),
                };
                terminal.draw(|f| {
                    let chunks = Layout::default()
                        .direction(Direction::Vertical)
                        .margin(2)
                        .constraints(
                            iter::repeat(Constraint::Length(1))
                                .take(args.hosts.len())
                                .chain(iter::once(Constraint::Percentage(10)))
                                .collect::<Vec<_>>()
                                .as_ref(),
                        )
                        .split(f.size());
                    for (((host_id, host), stats), &style) in args
                        .hosts
                        .iter()
                        .enumerate()
                        .zip(app.stats())
                        .zip(&app.styles)
                    {
                        let header_layout = Layout::default()
                            .direction(Direction::Horizontal)
                            .constraints(
                                [
                                    Constraint::Percentage(25),
                                    Constraint::Percentage(25),
                                    Constraint::Percentage(25),
                                    Constraint::Percentage(25),
                                ]
                                .as_ref(),
                            )
                            .split(chunks[host_id]);

                        f.render_widget(
                            Paragraph::new(format!("Pinging {}", host)).style(style),
                            header_layout[0],
                        );

                        f.render_widget(
                            Paragraph::new(format!(
                                "min {:?}",
                                Duration::from_micros(stats.minimum().unwrap_or(0))
                            ))
                            .style(style),
                            header_layout[1],
                        );
                        f.render_widget(
                            Paragraph::new(format!(
                                "max {:?}",
                                Duration::from_micros(stats.maximum().unwrap_or(0))
                            ))
                            .style(style),
                            header_layout[2],
                        );
                        f.render_widget(
                            Paragraph::new(format!(
                                "p95 {:?}",
                                Duration::from_micros(stats.percentile(95.0).unwrap_or(0))
                            ))
                            .style(style),
                            header_layout[3],
                        );
                    }

                    let datasets: Vec<_> = app
                        .data
                        .iter()
                        .zip(&app.styles)
                        .map(|(data, &style)| {
                            Dataset::default()
                                .marker(symbols::Marker::Braille)
                                .style(style)
                                .graph_type(GraphType::Line)
                                .data(data.as_slice())
                        })
                        .collect();

                    let y_axis_bounds = app.y_axis_bounds();

                    let chart = Chart::new(datasets)
                        .block(Block::default().borders(Borders::NONE))
                        .x_axis(
                            Axis::default()
                                .style(Style::default().fg(Color::Gray))
                                .bounds(app.x_axis_bounds()),
                        )
                        .y_axis(
                            Axis::default()
                                .style(Style::default().fg(Color::Gray))
                                .bounds(y_axis_bounds)
                                .labels(app.y_axis_labels(y_axis_bounds)),
                        );
                    f.render_widget(chart, chunks[args.hosts.len()]);
                })?;
            }
            Event::Input(input) => match input.code {
                KeyCode::Char('q') | KeyCode::Esc => {
                    killed.store(true, Ordering::Release);
                    break;
                }
                KeyCode::Char('c') if input.modifiers == KeyModifiers::CONTROL => {
                    killed.store(true, Ordering::Release);
                    break;
                }
                _ => {}
            },
        }
    }

    for thread in threads {
        thread.join().unwrap()?;
    }

    disable_raw_mode()?;
    execute!(
        terminal.backend_mut(),
        LeaveAlternateScreen,
        DisableMouseCapture
    )?;
    terminal.show_cursor()?;

    Ok(())
}
<|endoftext|>


<commit_before>fn main() {
    println!("Hello, world!");
}

#[test]
fn it_works() {
}

pub fn add_two(a: i32) -> i32 {
    a + 2
}

#[test]
fn it_works() {
    assert_eq!(4, add_two(2));
}
<commit_msg>removed duplicate function<commit_after>fn main() {
    println!("Hello, world!");
}

pub fn add_two(a: i32) -> i32 {
    a + 2
}

#[test]
fn it_works() {
    assert_eq!(4, add_two(2));
}
<|endoftext|>


<commit_before>extern crate rustless;
extern crate hyper;
extern crate iron;
extern crate valico;
extern crate crypto;

#[macro_use(bson, doc)]
extern crate bson;
extern crate mongodb;

use mongodb::{Client, ThreadedClient};
use mongodb::db::ThreadedDatabase;
use bson::
{
    Bson,
    Document
};

use std::fs;
use std::fs::File;
use std::io::BufReader;
use std::io::Read;
use std::path::Path;
use std::process::Command;
use std::thread;
use std::time::
{
    SystemTime,
    UNIX_EPOCH
};
use crypto::digest::Digest;
use crypto::sha3::Sha3;
use valico::json_dsl;
use rustless::server::status::StatusCode;
use rustless::
{
    Application,
    Api,
    Nesting,
    Versioning
};
use hyper::header::
{
    ContentDisposition,
    DispositionType,
    DispositionParam,
    Charset
};

const GBS_DIR:&str = "/var/lib/gbs";

fn check(path:&str) -> Option<&str>
{
    let p = Path::new(path).file_name()?.to_str()?;
    if p == path
    {
        return Some(path);
    }
    else
    {
        return None;
    }
}

fn is_atom(repository:&str, category:&str, package:&str, version:&str) -> Option<()>
{

    let r = check(repository)?;
    let c = check(category)?;
    let p = check(package)?;
    let v = check(version)?;

    if Path::new(&format!("{}/repos/{}/{}/{}/{}-{}.ebuild", GBS_DIR, r, c, p, p, v)).is_file()
    {
        return Some(());
    }
    else
    {
        return None;
    }
}

fn is_build_request(repository:&str, category:&str, package:&str, version:&str, id:&str) -> Option<()>
{

    let r = check(repository)?;
    let c = check(category)?;
    let p = check(package)?;
    let v = check(version)?;
    let i = check(id)?;

    if Path::new(&format!("{}/packages/{}/{}/{}/{}/{}", GBS_DIR, r, c, p, v, i)).is_dir()
    {
        return Some(());
    }
    else
    {
        return None;
    }
}

fn build_id(repository:&str, category:&str, package:&str, version:&str, uses:&[(String, bool)]) -> String
{
    let mut hasher = Sha3::sha3_256();
    hasher.input_str(&format!("{}{}{}{}", repository, category, package, version));

    for &(ref key, ref value) in uses.iter()
    {
        hasher.input_str(&format!("{}{} ", match *value {true => "", false => "-"}, key));
    }
    return hasher.result_str();
}

#[test]
fn test_is_atom()
{
    assert_eq!(false, is_atom("test","test","test","test").is_some());
    assert_eq!(false, is_atom("test/.","test/../.","test","test").is_some());
    assert_eq!(false, is_atom("gentoo/.","app-editors","vim","8.0.1298").is_some());
    assert_eq!(false, is_atom("gentoo","app-editors","vim","8").is_some());
    assert_eq!(true, is_atom("gentoo","app-editors","vim","8.0.1298").is_some());
}

fn main()
{
    let client = Client::connect("localhost", 27017)
        .expect("Failed to initialize standalone client.");
    let coll = client.db("gbs").collection("builds");

    let api = Api::build(|api|
    {
        api.prefix("api");
        api.version("1", Versioning::Path);

        api.namespace("atoms/:repositories/:categories/:packages/:versions", |atoms_ns|
        {
            atoms_ns.params(|params|
            {
                params.req_typed("repositories", json_dsl::string());
                params.req_typed("categories", json_dsl::string());
                params.req_typed("packages", json_dsl::string());
                params.req_typed("versions", json_dsl::string());
            });
            atoms_ns.get("builds/:id/status", |endpoint|
            {
                endpoint.params(|params|
                {
                    params.req_typed("id", json_dsl::string());
                });
                endpoint.handle(|mut client, params|
                {
                    let repository = params.find("repositories").unwrap().to_string().trim_matches('"').to_string();
                    let category = params.find("categories").unwrap().to_string().trim_matches('"').to_string();
                    let package = params.find("packages").unwrap().to_string().trim_matches('"').to_string();
                    let version = params.find("versions").unwrap().to_string().trim_matches('"').to_string();
                    let id = params.find("id").unwrap().to_string().trim_matches('"').to_string();

                    if is_build_request(&repository, &category, &package, &version, &id).is_some()
                    {
                        let s = format!("{}/packages/{}/{}/{}/{}/{}/status", GBS_DIR, repository, category, package, version, id);
                        let path = Path::new(&s);
                        if path.is_file()
                        {
                            let f = File::open(&s).unwrap();
                            let mut reader = BufReader::new(f);
                            let mut buf = String::new();
                            reader.read_to_string(&mut buf);
                            return client.text(buf);
                        }
                    }
                    client.set_status(StatusCode::NotFound);
                    return client.empty();
                })
            });
            atoms_ns.get("builds/:id/log", |endpoint|
            {
                endpoint.params(|params|
                {
                    params.req_typed("id", json_dsl::string());
                });
                endpoint.handle(|mut client, params|
                {
                    let repository = params.find("repositories").unwrap().to_string().trim_matches('"').to_string();
                    let category = params.find("categories").unwrap().to_string().trim_matches('"').to_string();
                    let package = params.find("packages").unwrap().to_string().trim_matches('"').to_string();
                    let version = params.find("versions").unwrap().to_string().trim_matches('"').to_string();
                    let id = params.find("id").unwrap().to_string().trim_matches('"').to_string();

                    if is_build_request(&repository, &category, &package, &version, &id).is_some()
                    {
                        let s = format!("{}/packages/{}/{}/{}/{}/{}/log", GBS_DIR, repository, category, package, version, id);
                        let path = Path::new(&s);
                        if path.is_file()
                        {
                            let f = File::open(&s).unwrap();
                            let mut reader = BufReader::new(f);
                            let mut buf = String::new();
                            reader.read_to_string(&mut buf);
                            return client.text(buf);
                        }
                    }
                    client.set_status(StatusCode::NotFound);
                    return client.empty();
                })
            });
            atoms_ns.get("builds/:id", |endpoint|
            {
                endpoint.params(|params|
                {
                    params.req_typed("id", json_dsl::string());
                });
                endpoint.handle(|mut client, params|
                {
                    let repository = params.find("repositories").unwrap().to_string().trim_matches('"').to_string();
                    let category = params.find("categories").unwrap().to_string().trim_matches('"').to_string();
                    let package = params.find("packages").unwrap().to_string().trim_matches('"').to_string();
                    let version = params.find("versions").unwrap().to_string().trim_matches('"').to_string();
                    let id = params.find("id").unwrap().to_string().trim_matches('"').to_string();

                    if is_build_request(&repository, &category, &package, &version, &id).is_some()
                    {
                        let s = format!("{}/packages/{}/{}/{}/{}/{}/{3}-{4}.tbz2", GBS_DIR, repository, category, package, version, id);
                        let path = Path::new(&s);
                        if path.is_file()
                        {
                            client.set_header(ContentDisposition
                            {
                              disposition: DispositionType::Attachment,
                              parameters: vec![DispositionParam::Filename(
                                Charset::Us_Ascii,
                                None,
                                format!("{}-{}.tbz2", package, version).into_bytes()
                            )]});
                            return client.file(path);
                        }
                    }
                    client.set_status(StatusCode::NotFound);
                    return client.empty();
                })
            });
            atoms_ns.get("builds", |endpoint|
            {
                endpoint.handle(|mut client, params|
                {
                    let repository = params.find("repositories").unwrap().to_string().trim_matches('"').to_string();
                    let category = params.find("categories").unwrap().to_string().trim_matches('"').to_string();
                    let package = params.find("packages").unwrap().to_string().trim_matches('"').to_string();
                    let version = params.find("versions").unwrap().to_string().trim_matches('"').to_string();
                    let use_flag = params.find("use").unwrap();

                    let mut uu :Vec<(String, bool)> = Vec::new();
                    for (key, value) in use_flag.as_object().unwrap().iter()
                    {
                        uu.push((key.clone(), value.as_bool().unwrap()));
                    }

                    let id = build_id(&repository, &category, &package, &version, &uu);

                    if is_build_request(&repository, &category, &package, &version, &id).is_some()
                    {
                        let url = format!("{}/{}/{}/{}/builds/{}", repository, category, package, version, id);
                        return client.text(url);
                    }
                    else
                    {
                        client.set_status(StatusCode::NotFound);
                        return client.empty();
                    }
                })
            });
            atoms_ns.post("builds", |endpoint|
            {
                endpoint.handle(move |mut client, params|
                {
                    let repository = params.find("repositories").unwrap().to_string().trim_matches('"').to_string();
                    let category = params.find("categories").unwrap().to_string().trim_matches('"').to_string();
                    let package = params.find("packages").unwrap().to_string().trim_matches('"').to_string();
                    let version = params.find("versions").unwrap().to_string().trim_matches('"').to_string();
                    let use_flag = params.find("use").unwrap();

                    let mut uses = String::new();
                    for (key, value) in use_flag.as_object().unwrap().iter()
                    {
                        uses.push_str(&format!("{}{} ", match value.as_bool().unwrap(){true => "", false => "-"}, key));
                    }
                    let mut uu :Vec<(String, bool)> = Vec::new();
                    for (key, value) in use_flag.as_object().unwrap().iter()
                    {
                        uu.push((key.clone(), value.as_bool().unwrap()));
                    }
                    let id = build_id(&repository, &category, &package, &version, &uu);
                    println!("{}/{}/{}/{}", repository, category, package, version);
                    println!("USE=\"{}\"", uses);

                    let url = format!("{}/{}/{}/{}/builds/{}", repository, category, package, version, id);

                    if is_atom(&repository, &category, &package, &version).is_some()
                    {
                        if !is_build_request(&repository, &category, &package, &version, &id).is_some()
                        {
                            {
                                let mut uses = Document::new();
                                for (key, value) in use_flag.as_object().unwrap().iter()
                                {
                                    uses.insert(key.clone(), Bson::Boolean(value.as_bool().unwrap().clone()));
                                }
                                let s = SystemTime::now();
                                let doc = doc!
                                {
                                    "date" => &format!("{}", s.duration_since(UNIX_EPOCH).unwrap().as_secs()),
                                    "id" => id.clone(),
                                    "repository" => repository.clone(),
                                    "category" => category.clone(),
                                    "package" => package.clone(),
                                    "version" => version.clone(),
                                    "use" => uses
                                };
                                coll.insert_one(doc.clone(), None).ok().expect("Failed to insert document.");
                            }
                            thread::spawn(move ||
                            {
                                fs::create_dir_all(format!("{}/packages/{}/{}/{}/{}/{}", GBS_DIR, repository, category, package, version, id)).unwrap();
                                let b = Command::new("docker")
                                .arg("run")
                                .arg("--rm")
                                .arg("-v")
                                .arg(format!("{}/distfiles:/usr/portage/distfiles", GBS_DIR))
                                .arg("-v")
                                .arg(format!("{}/repos:/var/db/repos:ro", GBS_DIR))
                                .arg("-v")
                                .arg(format!("{}/ccache:/mnt/ccache", GBS_DIR))
                                .arg("-v")
                                .arg(format!("{}/packages/{}/{}/{}/{}/{}:/mnt/package", GBS_DIR, repository, category, package, version, id))
                                .arg("--cap-add=SYS_PTRACE")
                                .arg("-idt")
                                .arg("--name")
                                .arg(format!("{}_{}_{}_{}_{}", repository, category, package, version, id))
                                .arg("gentoo:gbs")
                                .arg("/usr/bin/build_script.sh")
                                .arg(repository)
                                .arg(category)
                                .arg(package)
                                .arg(version)
                                .arg(uses)
                                .output().unwrap();
                                println!("{}\n", String::from_utf8_lossy(&b.stdout));
                                println!("{}\n", String::from_utf8_lossy(&b.stderr));
                            });
                        }
                        return client.text(url);
                    }
                    else
                    {
                        client.set_status(StatusCode::NotFound);
                        return client.empty();
                    }
                })
            });
        });
    });

    let app = Application::new(api);

    iron::Iron::new(app).http("0.0.0.0:4000").unwrap();
}
<commit_msg>cosme log<commit_after>extern crate rustless;
extern crate hyper;
extern crate iron;
extern crate valico;
extern crate crypto;

#[macro_use(bson, doc)]
extern crate bson;
extern crate mongodb;

use mongodb::{Client, ThreadedClient};
use mongodb::db::ThreadedDatabase;
use bson::
{
    Bson,
    Document
};

use std::fs;
use std::fs::File;
use std::io::BufReader;
use std::io::Read;
use std::path::Path;
use std::process::Command;
use std::thread;
use std::time::
{
    SystemTime,
    UNIX_EPOCH
};
use crypto::digest::Digest;
use crypto::sha3::Sha3;
use valico::json_dsl;
use rustless::server::status::StatusCode;
use rustless::
{
    Application,
    Api,
    Nesting,
    Versioning
};
use hyper::header::
{
    ContentDisposition,
    DispositionType,
    DispositionParam,
    Charset
};

const GBS_DIR:&str = "/var/lib/gbs";

fn check(path:&str) -> Option<&str>
{
    let p = Path::new(path).file_name()?.to_str()?;
    if p == path
    {
        return Some(path);
    }
    else
    {
        return None;
    }
}

fn is_atom(repository:&str, category:&str, package:&str, version:&str) -> Option<()>
{

    let r = check(repository)?;
    let c = check(category)?;
    let p = check(package)?;
    let v = check(version)?;

    if Path::new(&format!("{}/repos/{}/{}/{}/{}-{}.ebuild", GBS_DIR, r, c, p, p, v)).is_file()
    {
        return Some(());
    }
    else
    {
        return None;
    }
}

fn is_build_request(repository:&str, category:&str, package:&str, version:&str, id:&str) -> Option<()>
{

    let r = check(repository)?;
    let c = check(category)?;
    let p = check(package)?;
    let v = check(version)?;
    let i = check(id)?;

    if Path::new(&format!("{}/packages/{}/{}/{}/{}/{}", GBS_DIR, r, c, p, v, i)).is_dir()
    {
        return Some(());
    }
    else
    {
        return None;
    }
}

fn build_id(repository:&str, category:&str, package:&str, version:&str, uses:&[(String, bool)]) -> String
{
    let mut hasher = Sha3::sha3_256();
    hasher.input_str(&format!("{}{}{}{}", repository, category, package, version));

    for &(ref key, ref value) in uses.iter()
    {
        hasher.input_str(&format!("{}{} ", match *value {true => "", false => "-"}, key));
    }
    return hasher.result_str();
}

#[test]
fn test_is_atom()
{
    assert_eq!(false, is_atom("test","test","test","test").is_some());
    assert_eq!(false, is_atom("test/.","test/../.","test","test").is_some());
    assert_eq!(false, is_atom("gentoo/.","app-editors","vim","8.0.1298").is_some());
    assert_eq!(false, is_atom("gentoo","app-editors","vim","8").is_some());
    assert_eq!(true, is_atom("gentoo","app-editors","vim","8.0.1298").is_some());
}

fn main()
{
    let client = Client::connect("localhost", 27017)
        .expect("Failed to initialize standalone client.");
    let coll = client.db("gbs").collection("builds");

    let api = Api::build(|api|
    {
        api.prefix("api");
        api.version("1", Versioning::Path);

        api.namespace("atoms/:repositories/:categories/:packages/:versions", |atoms_ns|
        {
            atoms_ns.params(|params|
            {
                params.req_typed("repositories", json_dsl::string());
                params.req_typed("categories", json_dsl::string());
                params.req_typed("packages", json_dsl::string());
                params.req_typed("versions", json_dsl::string());
            });
            atoms_ns.get("builds/:id/status", |endpoint|
            {
                endpoint.params(|params|
                {
                    params.req_typed("id", json_dsl::string());
                });
                endpoint.handle(|mut client, params|
                {
                    let repository = params.find("repositories").unwrap().to_string().trim_matches('"').to_string();
                    let category = params.find("categories").unwrap().to_string().trim_matches('"').to_string();
                    let package = params.find("packages").unwrap().to_string().trim_matches('"').to_string();
                    let version = params.find("versions").unwrap().to_string().trim_matches('"').to_string();
                    let id = params.find("id").unwrap().to_string().trim_matches('"').to_string();

                    if is_build_request(&repository, &category, &package, &version, &id).is_some()
                    {
                        let s = format!("{}/packages/{}/{}/{}/{}/{}/status", GBS_DIR, repository, category, package, version, id);
                        let path = Path::new(&s);
                        if path.is_file()
                        {
                            let f = File::open(&s).unwrap();
                            let mut reader = BufReader::new(f);
                            let mut buf = String::new();
                            reader.read_to_string(&mut buf);
                            return client.text(buf);
                        }
                    }
                    client.set_status(StatusCode::NotFound);
                    return client.empty();
                })
            });
            atoms_ns.get("builds/:id/log", |endpoint|
            {
                endpoint.params(|params|
                {
                    params.req_typed("id", json_dsl::string());
                });
                endpoint.handle(|mut client, params|
                {
                    let repository = params.find("repositories").unwrap().to_string().trim_matches('"').to_string();
                    let category = params.find("categories").unwrap().to_string().trim_matches('"').to_string();
                    let package = params.find("packages").unwrap().to_string().trim_matches('"').to_string();
                    let version = params.find("versions").unwrap().to_string().trim_matches('"').to_string();
                    let id = params.find("id").unwrap().to_string().trim_matches('"').to_string();

                    if is_build_request(&repository, &category, &package, &version, &id).is_some()
                    {
                        let s = format!("{}/packages/{}/{}/{}/{}/{}/log", GBS_DIR, repository, category, package, version, id);
                        let path = Path::new(&s);
                        if path.is_file()
                        {
                            let f = File::open(&s).unwrap();
                            let mut reader = BufReader::new(f);
                            let mut buf = String::new();
                            reader.read_to_string(&mut buf);
                            return client.text(buf);
                        }
                    }
                    client.set_status(StatusCode::NotFound);
                    return client.empty();
                })
            });
            atoms_ns.get("builds/:id", |endpoint|
            {
                endpoint.params(|params|
                {
                    params.req_typed("id", json_dsl::string());
                });
                endpoint.handle(|mut client, params|
                {
                    let repository = params.find("repositories").unwrap().to_string().trim_matches('"').to_string();
                    let category = params.find("categories").unwrap().to_string().trim_matches('"').to_string();
                    let package = params.find("packages").unwrap().to_string().trim_matches('"').to_string();
                    let version = params.find("versions").unwrap().to_string().trim_matches('"').to_string();
                    let id = params.find("id").unwrap().to_string().trim_matches('"').to_string();

                    if is_build_request(&repository, &category, &package, &version, &id).is_some()
                    {
                        let s = format!("{}/packages/{}/{}/{}/{}/{}/{3}-{4}.tbz2", GBS_DIR, repository, category, package, version, id);
                        let path = Path::new(&s);
                        if path.is_file()
                        {
                            client.set_header(ContentDisposition
                            {
                              disposition: DispositionType::Attachment,
                              parameters: vec![DispositionParam::Filename(
                                Charset::Us_Ascii,
                                None,
                                format!("{}-{}.tbz2", package, version).into_bytes()
                            )]});
                            return client.file(path);
                        }
                    }
                    client.set_status(StatusCode::NotFound);
                    return client.empty();
                })
            });
            atoms_ns.get("builds", |endpoint|
            {
                endpoint.handle(|mut client, params|
                {
                    let repository = params.find("repositories").unwrap().to_string().trim_matches('"').to_string();
                    let category = params.find("categories").unwrap().to_string().trim_matches('"').to_string();
                    let package = params.find("packages").unwrap().to_string().trim_matches('"').to_string();
                    let version = params.find("versions").unwrap().to_string().trim_matches('"').to_string();
                    let use_flag = params.find("use").unwrap();

                    let mut uses = String::new();
                    for (key, value) in use_flag.as_object().unwrap().iter()
                    {
                        uses.push_str(&format!("{}{} ", match value.as_bool().unwrap(){true => "", false => "-"}, key));
                    }

                    let mut uu :Vec<(String, bool)> = Vec::new();
                    for (key, value) in use_flag.as_object().unwrap().iter()
                    {
                        uu.push((key.clone(), value.as_bool().unwrap()));
                    }

                    let id = build_id(&repository, &category, &package, &version, &uu);

                    println!("GET {}/{}/{}/{}", repository, category, package, version);
                    println!("USE=\"{}\"", uses);
                    println!("id:{}", id);

                    if is_build_request(&repository, &category, &package, &version, &id).is_some()
                    {
                        println!("Available!");
                        println!("");
                        let url = format!("{}/{}/{}/{}/builds/{}", repository, category, package, version, id);
                        return client.text(url);
                    }
                    else
                    {
                        println!("N/A");
                        println!("");
                        client.set_status(StatusCode::NotFound);
                        return client.empty();
                    }
                })
            });
            atoms_ns.post("builds", |endpoint|
            {
                endpoint.handle(move |mut client, params|
                {
                    let repository = params.find("repositories").unwrap().to_string().trim_matches('"').to_string();
                    let category = params.find("categories").unwrap().to_string().trim_matches('"').to_string();
                    let package = params.find("packages").unwrap().to_string().trim_matches('"').to_string();
                    let version = params.find("versions").unwrap().to_string().trim_matches('"').to_string();
                    let use_flag = params.find("use").unwrap();

                    let mut uses = String::new();
                    for (key, value) in use_flag.as_object().unwrap().iter()
                    {
                        uses.push_str(&format!("{}{} ", match value.as_bool().unwrap(){true => "", false => "-"}, key));
                    }
                    let mut uu :Vec<(String, bool)> = Vec::new();
                    for (key, value) in use_flag.as_object().unwrap().iter()
                    {
                        uu.push((key.clone(), value.as_bool().unwrap()));
                    }
                    let id = build_id(&repository, &category, &package, &version, &uu);
                    println!("POST {}/{}/{}/{}", repository, category, package, version);
                    println!("USE=\"{}\"", uses);
                    println!("id:{}", id);
                    println!("");

                    let url = format!("{}/{}/{}/{}/builds/{}", repository, category, package, version, id);

                    if is_atom(&repository, &category, &package, &version).is_some()
                    {
                        if !is_build_request(&repository, &category, &package, &version, &id).is_some()
                        {
                            {
                                let mut uses = Document::new();
                                for (key, value) in use_flag.as_object().unwrap().iter()
                                {
                                    uses.insert(key.clone(), Bson::Boolean(value.as_bool().unwrap().clone()));
                                }
                                let s = SystemTime::now();
                                let doc = doc!
                                {
                                    "date" => &format!("{}", s.duration_since(UNIX_EPOCH).unwrap().as_secs()),
                                    "id" => id.clone(),
                                    "repository" => repository.clone(),
                                    "category" => category.clone(),
                                    "package" => package.clone(),
                                    "version" => version.clone(),
                                    "use" => uses
                                };
                                coll.insert_one(doc.clone(), None).ok().expect("Failed to insert document.");
                            }
                            thread::spawn(move ||
                            {
                                fs::create_dir_all(format!("{}/packages/{}/{}/{}/{}/{}", GBS_DIR, repository, category, package, version, id)).unwrap();
                                let b = Command::new("docker")
                                .arg("run")
                                .arg("--rm")
                                .arg("-v")
                                .arg(format!("{}/distfiles:/usr/portage/distfiles", GBS_DIR))
                                .arg("-v")
                                .arg(format!("{}/repos:/var/db/repos:ro", GBS_DIR))
                                .arg("-v")
                                .arg(format!("{}/ccache:/mnt/ccache", GBS_DIR))
                                .arg("-v")
                                .arg(format!("{}/packages/{}/{}/{}/{}/{}:/mnt/package", GBS_DIR, repository, category, package, version, id))
                                .arg("--cap-add=SYS_PTRACE")
                                .arg("-idt")
                                .arg("--name")
                                .arg(format!("{}_{}_{}_{}_{}", repository, category, package, version, id))
                                .arg("gentoo:gbs")
                                .arg("/usr/bin/build_script.sh")
                                .arg(repository)
                                .arg(category)
                                .arg(package)
                                .arg(version)
                                .arg(uses)
                                .output().unwrap();
                                println!("{}\n", String::from_utf8_lossy(&b.stdout));
                                println!("{}\n", String::from_utf8_lossy(&b.stderr));
                            });
                        }
                        return client.text(url);
                    }
                    else
                    {
                        client.set_status(StatusCode::NotFound);
                        return client.empty();
                    }
                })
            });
        });
    });

    let app = Application::new(api);

    iron::Iron::new(app).http("0.0.0.0:4000").unwrap();
}
<|endoftext|>


<commit_before>extern crate clap;
extern crate md5;
extern crate walkdir;

use clap::{Arg, App};
use md5::{Digest};
use std::cmp;
use std::fmt::{Display, Formatter, Result};
use std::fs::{File, Metadata};
use std::io::Read;
use std::path::Path;
use walkdir::WalkDir;

fn main() {
    let matches = App::new("T_Rust")
        .version("1.0")
        .author("Adam Childs <adam.childs@gmail.com>")
        .about("")
        .arg(Arg::with_name("DIRECTORY")
            .help("Sets the input directory to use")
            .required(true)
            .index(1))
        .get_matches();

    let dir: &str = matches.value_of("DIRECTORY").unwrap_or("/Users/adam.childs/Desktop");
    println!("Dir: {}", dir);

    let mut total_size: f64 = 0.0;
    for entry in WalkDir::new(dir).into_iter().filter_map(|e| e.ok()) {
        let file: File = File::open(entry.path()).unwrap();
        let metadata: Metadata = file.metadata().unwrap();
        let file_path: String = String::from(entry.path().to_str().unwrap());
        let sys_file: SystemFile = SystemFile::from(file_path);

        total_size += metadata.len() as f64;

        println!("{}", sys_file);
    }

    println!("Total size: {}", SystemFile::human_readable_size(total_size));
}

///
/// Simple representation of a File on the user's system.
///
struct SystemFile {
    name: String,
    path: String,
    size: String,
    md5: Digest,
    is_dir: bool
}

impl SystemFile {

    ///
    /// Constructs a new `SystemFile` from the given arguments.
    ///
    pub fn from(path: String) -> SystemFile {
        let mut file: File = File::open(&path).unwrap();
        let metadata: Metadata = file.metadata().unwrap();

        let mut buffer: Vec<u8> = Vec::new();
        if !metadata.is_dir() {
            // Read the entire file to a buffer
            file.read_to_end(&mut buffer).unwrap();
        }

        SystemFile {
            name: String::from(Path::new(&path).file_name().unwrap().to_str().unwrap()),
            path: path,
            size: SystemFile::human_readable_size(metadata.len() as f64),
            md5: md5::compute(buffer),
            is_dir: metadata.is_dir()
        }
    }

    ///
    /// Returns the human readable representation of the current files size in bytes as a `String`.
    ///
    pub fn human_readable_size(num: f64) -> String {
        let negative = if num.is_sign_positive() { "" } else { "-" };
        let num = num.abs();
        let units = ["bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
        if num < 1_f64 {
            return format!("{}{} {}", negative, num, "B");
        }
        let delimiter = 1000_f64;
        let exponent = cmp::min((num.ln() / delimiter.ln()).floor() as i32, (units.len() - 1) as i32);
        let pretty_bytes = format!("{:.2}", num / delimiter.powi(exponent)).parse::<f64>().unwrap() * 1_f64;
        let unit = units[exponent as usize];
        format!("{}{} {}", negative, pretty_bytes, unit)
    }

}

impl Display for SystemFile {
    fn fmt(&self, f: &mut Formatter) -> Result {
        write!(f, "({}, {}, {}, {})", format!("{:x}", self.md5), self.name, self.size, self.path)
    }
}<commit_msg>Prefer constants for program argument setup.<commit_after>extern crate clap;
extern crate md5;
extern crate walkdir;

use clap::{Arg, App};
use md5::{Digest};
use std::cmp;
use std::fmt::{Display, Formatter, Result};
use std::fs::{File, Metadata};
use std::io::Read;
use std::path::Path;
use walkdir::WalkDir;

const NAME: &'static str = "TrustAntivirus";
const VERSION: &'static str = "1.0";
const AUTHOR: &'static str = "Adam Childs <adamdchilds@gmail.com>";
const ABOUT: &'static str = "";

fn main() {
    let matches = App::new(NAME)
        .version(VERSION)
        .author(AUTHOR)
        .about(ABOUT)
        .arg(Arg::with_name("DIRECTORY")
            .help("Sets the input directory to use")
            .required(true)
            .index(1))
        .get_matches();

    let dir: &str = matches.value_of("DIRECTORY").unwrap_or("/Users/adam.childs/Desktop");
    println!("Dir: {}", dir);

    let mut total_size: f64 = 0.0;
    for entry in WalkDir::new(dir).into_iter().filter_map(|e| e.ok()) {
        let file: File = File::open(entry.path()).unwrap();
        let metadata: Metadata = file.metadata().unwrap();
        let file_path: String = String::from(entry.path().to_str().unwrap());
        let sys_file: SystemFile = SystemFile::from(file_path);

        total_size += metadata.len() as f64;

        println!("{}", sys_file);
    }

    println!("Total size: {}", SystemFile::human_readable_size(total_size));
}

///
/// Simple representation of a File on the user's system.
///
struct SystemFile {
    name: String,
    path: String,
    size: String,
    md5: Digest,
    is_dir: bool
}

impl SystemFile {

    ///
    /// Constructs a new `SystemFile` from the given arguments.
    ///
    pub fn from(path: String) -> SystemFile {
        let mut file: File = File::open(&path).unwrap();
        let metadata: Metadata = file.metadata().unwrap();

        let mut buffer: Vec<u8> = Vec::new();
        if !metadata.is_dir() {
            // Read the entire file to a buffer
            file.read_to_end(&mut buffer).unwrap();
        }

        SystemFile {
            name: String::from(Path::new(&path).file_name().unwrap().to_str().unwrap()),
            path: path,
            size: SystemFile::human_readable_size(metadata.len() as f64),
            md5: md5::compute(buffer),
            is_dir: metadata.is_dir()
        }
    }

    ///
    /// Returns the human readable representation of the current files size in bytes as a `String`.
    ///
    pub fn human_readable_size(num: f64) -> String {
        let negative = if num.is_sign_positive() { "" } else { "-" };
        let num = num.abs();
        let units = ["bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
        if num < 1_f64 {
            return format!("{}{} {}", negative, num, "B");
        }
        let delimiter = 1000_f64;
        let exponent = cmp::min((num.ln() / delimiter.ln()).floor() as i32, (units.len() - 1) as i32);
        let pretty_bytes = format!("{:.2}", num / delimiter.powi(exponent)).parse::<f64>().unwrap() * 1_f64;
        let unit = units[exponent as usize];
        format!("{}{} {}", negative, pretty_bytes, unit)
    }

}

impl Display for SystemFile {
    fn fmt(&self, f: &mut Formatter) -> Result {
        write!(f, "({}, {}, {}, {})", format!("{:x}", self.md5), self.name, self.size, self.path)
    }
}<|endoftext|>


<commit_before>// Copyright 2015 Robert Helmer <rhelmer@rhelmer.org>. See the LICENSE
// file at the top-level directory of this distribution.

/**
  * Symbolapi - a microservice which accepts lists of symbol+addresses, and returns
  * a list of symbolicated functions.
  *
  * The goal is to take an HTTP JSON request such as:
  * {"stacks":[
  *   [
  *     [0,11723767],
  *     [1, 65802]
  *    ]
  *  ],
  *  "memoryMap":[
  *    ["xul.pdb","44E4EC8C2F41492B9369D6B9A059577C2"],
  *    ["wntdll.pdb","D74F79EB1F8D4A45ABCD2F476CCABACC2"]
  *  ],
  *  "version":4
  * }
  *
  * The symbolapi service then downloads the corresponding symbol files (e.g. from S3), and returns
  * the function names for the corresponding addresses (in the "stacks" array) and returns JSON
  * such as:
  * {"symbolicatedStacks": [
  *   [
  *     "XREMain::XRE_mainRun() (in xul.pdb)",
  *     "KiUserCallbackDispatcher (in wntdll.pdb)"]
  *   ],
  *   "knownModules": [true, true]
  * }
  */

extern crate breakpad_symbols;
extern crate hyper;
#[macro_use]
extern crate log;
extern crate log4rs;
extern crate rustc_serialize;
extern crate toml;

use std::collections::HashMap;
use std::fs::{File, create_dir_all};
use std::io::{Read, Write};
use std::path::{PathBuf};
use std::thread;

use breakpad_symbols::SymbolFile;
use hyper::Client;
use hyper::server::{Server, Request, Response};
use hyper::status::StatusCode;
use rustc_serialize::json;

/**
  * Incoming JSON request format.
  */
// required JSON keys are non-snakecase
#[allow(non_snake_case)]
#[derive(RustcDecodable)]
pub struct SymbolRequest {
    memoryMap: Vec<(String,String)>,
    // index, offset
    stacks: Vec<Vec<(i8,u64)>>,
    version: u8,
}

/**
  * Outgoing JSON response format.
  */
// required JSON keys are non-snakecase
#[allow(non_snake_case)]
#[derive(RustcEncodable)]
pub struct SymbolResponse {
    symbolicatedStacks: Vec<Vec<String>>,
    knownModules: Vec<bool>,
}

fn main() {
    log4rs::init_file("config/log.toml", Default::default()).unwrap();

    let address = "0.0.0.0:8080";

    info!("Listening on {}", address);
    Server::http(address).unwrap().handle(server).unwrap();
}

/**
  * Receives single HTTP requests and demuxes to symbols file fetches from S3 bucket.
  */
fn server(mut req: Request, mut res: Response) {
    info!("incoming connection from {}", req.remote_addr);

    match req.method {
        hyper::Post => {
            let mut res = res.start().unwrap();

            let mut buffer = String::new();
            let _ = req.read_to_string(&mut buffer);
            debug!("raw POST: {:?}", &buffer);

            let decoded: SymbolRequest = json::decode(&buffer).unwrap();

            debug!("decoded memoryMap: {:?}", decoded.memoryMap);
            debug!("decoded stacks: {:?}", decoded.stacks);
            debug!("decoded version: {}", decoded.version);

            let symbol_url = get_config("symbol_urls.public");

            // stacks come in as an array, turn into hashmap
            let mut stack_map: HashMap<i8, Vec<u64>> = HashMap::new();

            debug!("decoded.stacks: {:?}", decoded.stacks);

            let stacks = decoded.stacks[0].clone();
            for stack in &stacks {
                let (index, offset) = *stack;

                let mut offsets = vec!();
                if stack_map.contains_key(&index) {
                    offsets = stack_map.get(&index).unwrap().clone();
                }
                offsets.push(offset);
                stack_map.insert(index, offsets);
            }

            debug!("stack_map: {:?}", stack_map);

            let symbol_response = client(symbol_url, decoded.memoryMap, stack_map.clone());
            let _ = res.write_all(symbol_response.as_bytes());

            res.end().unwrap();
        },
        _ => { *res.status_mut() = StatusCode::MethodNotAllowed },
    }

    debug!("finished serving request");
}

/**
  * Creates multiple client connections and aggregates result.
  */
fn client(url: String, memory_map: Vec<(String,String)>, stack_map: HashMap<i8, Vec<u64>>) -> String {
    let mut handles = vec![];
    let mut counter: i8 = 0;
    for (debug_file, debug_id) in memory_map {
        let debug_file_name = debug_file.clone();
        let pdb = debug_file.find(".pdb").unwrap();
        let (symbol_name, _) = debug_file.split_at(pdb);
        let symbol_file = format!("{}.sym", symbol_name);
        let this_url = format!("{}/{}/{}/{}", url, debug_file, debug_id, symbol_file);

        let mut symbol_path = PathBuf::new();
        // TODO get from config
        symbol_path.push("testdata/symbols");
        symbol_path.push(&debug_file);
        symbol_path.push(&debug_id);
        symbol_path.push(&symbol_file);

        let stack_map_copy = stack_map.clone();

        let _ = create_dir_all(&symbol_path.parent().unwrap()).unwrap();

        // TODO most of the time is probably spent waiting on I/O, maybe async would be more appropriate?
        // TODO decide min/max possible threads, possibly based on number of cores?
        handles.push(thread::spawn(move || {
            // FIXME use Arc<Mutex<File>> to prevent concurrent writes
            // TODO only write contents if server version newer
            if !&symbol_path.exists() {

                let mut body = String::new();
                let c = Client::new();
                let mut res = c.get(&this_url).send().unwrap();
                let _ = res.read_to_string(&mut body);

                let mut f = File::create(&symbol_path).unwrap();
                let _ = f.write_all(body.as_bytes());
            }

            debug!("symbol_path: {:?}", &symbol_path);
            // FIXME this is having problems with CRLF
            let symbol_provider = match SymbolFile::from_file(&symbol_path) {
                Ok(x) => x,
                Err(x) => panic!("symbol provider failed: symbol_path: {:?}, {}", &symbol_path, x),
            };

            let mut symbols = vec!();
            for stacks in stack_map_copy.get(&counter) {
                for address in stacks {
                    debug!("attempt to symbolicate: {} for: {:?}", *address, &symbol_path);
                    match symbol_provider.functions.lookup(*address) {
                        Some(x) => symbols.push(x.name.clone()),
                        // return the address rather than function name if symbol not found
                        None => symbols.push(format!("0x{:x}", address)),
                    }
                }
            }
            debug!("debug_file_name: {:?}, symbols: {:?}", debug_file_name, symbols);

            (debug_file_name, symbols)
        }));

        counter += 1;
    }

    let mut result = SymbolResponse {
        symbolicatedStacks: vec!(),
        knownModules: vec!(),
    };

    for handle in handles {
        let (debug_file_name, symbols) = handle.join().unwrap();

        for symbol in symbols {
            result.symbolicatedStacks.push(
                vec!(format!("{} (in {}))", symbol, debug_file_name))
            );
        }
        result.knownModules.push(true);
    }

    json::encode(&result).unwrap()
}

/**
  * Returns individual values from the configuration file.
  */
fn get_config(value_name: &str) -> String {
    // TODO move to actual file, static str for the moment
    let toml: &'static str = r#"
        [symbol_urls]
        public = "https://s3-us-west-2.amazonaws.com/org.mozilla.crash-stats.symbols-public/v1"
    "#;
    // TODO support multiple URLs
    let value: toml::Value = toml.parse().unwrap();

    value.lookup(value_name).unwrap().as_str().unwrap().to_string()
}
<commit_msg>use the proper public function<commit_after>// Copyright 2015 Robert Helmer <rhelmer@rhelmer.org>. See the LICENSE
// file at the top-level directory of this distribution.

/**
  * Symbolapi - a microservice which accepts lists of symbol+addresses, and returns
  * a list of symbolicated functions.
  *
  * The goal is to take an HTTP JSON request such as:
  * {"stacks":[
  *   [
  *     [0,11723767],
  *     [1, 65802]
  *    ]
  *  ],
  *  "memoryMap":[
  *    ["xul.pdb","44E4EC8C2F41492B9369D6B9A059577C2"],
  *    ["wntdll.pdb","D74F79EB1F8D4A45ABCD2F476CCABACC2"]
  *  ],
  *  "version":4
  * }
  *
  * The symbolapi service then downloads the corresponding symbol files (e.g. from S3), and returns
  * the function names for the corresponding addresses (in the "stacks" array) and returns JSON
  * such as:
  * {"symbolicatedStacks": [
  *   [
  *     "XREMain::XRE_mainRun() (in xul.pdb)",
  *     "KiUserCallbackDispatcher (in wntdll.pdb)"]
  *   ],
  *   "knownModules": [true, true]
  * }
  */

extern crate breakpad_symbols;
extern crate hyper;
#[macro_use]
extern crate log;
extern crate log4rs;
extern crate rustc_serialize;
extern crate toml;

use std::collections::HashMap;
use std::fs::{File, create_dir_all};
use std::io::{Read, Write};
use std::path::{PathBuf};
use std::thread;

use breakpad_symbols::{Symbolizer, SimpleSymbolSupplier};
use hyper::Client;
use hyper::server::{Server, Request, Response};
use hyper::status::StatusCode;
use rustc_serialize::json;

/**
  * Incoming JSON request format.
  */
// required JSON keys are non-snakecase
#[allow(non_snake_case)]
#[derive(RustcDecodable)]
pub struct SymbolRequest {
    memoryMap: Vec<(String,String)>,
    // index, offset
    stacks: Vec<Vec<(i8,u64)>>,
    version: u8,
}

/**
  * Outgoing JSON response format.
  */
// required JSON keys are non-snakecase
#[allow(non_snake_case)]
#[derive(RustcEncodable)]
pub struct SymbolResponse {
    symbolicatedStacks: Vec<Vec<String>>,
    knownModules: Vec<bool>,
}

fn main() {
    log4rs::init_file("config/log.toml", Default::default()).unwrap();

    let address = "0.0.0.0:8080";

    info!("Listening on {}", address);
    Server::http(address).unwrap().handle(server).unwrap();
}

/**
  * Receives single HTTP requests and demuxes to symbols file fetches from S3 bucket.
  */
fn server(mut req: Request, mut res: Response) {
    info!("incoming connection from {}", req.remote_addr);

    match req.method {
        hyper::Post => {
            let mut res = res.start().unwrap();

            let mut buffer = String::new();
            let _ = req.read_to_string(&mut buffer);
            debug!("raw POST: {:?}", &buffer);

            let decoded: SymbolRequest = json::decode(&buffer).unwrap();

            debug!("decoded memoryMap: {:?}", decoded.memoryMap);
            debug!("decoded stacks: {:?}", decoded.stacks);
            debug!("decoded version: {}", decoded.version);

            let symbol_url = get_config("symbol_urls.public");

            // stacks come in as an array, turn into hashmap
            let mut stack_map: HashMap<i8, Vec<u64>> = HashMap::new();

            debug!("decoded.stacks: {:?}", decoded.stacks);

            let stacks = decoded.stacks[0].clone();
            for stack in &stacks {
                let (index, offset) = *stack;

                let mut offsets = vec!();
                if stack_map.contains_key(&index) {
                    offsets = stack_map.get(&index).unwrap().clone();
                }
                offsets.push(offset);
                stack_map.insert(index, offsets);
            }

            debug!("stack_map: {:?}", stack_map);

            let symbol_response = client(symbol_url, decoded.memoryMap, stack_map.clone());
            let _ = res.write_all(symbol_response.as_bytes());

            res.end().unwrap();
        },
        _ => { *res.status_mut() = StatusCode::MethodNotAllowed },
    }

    debug!("finished serving request");
}

/**
  * Creates multiple client connections and aggregates result.
  */
fn client(url: String, memory_map: Vec<(String,String)>, stack_map: HashMap<i8, Vec<u64>>) -> String {
    let mut handles = vec![];
    let mut counter: i8 = 0;
    for (debug_file, debug_id) in memory_map {
        let debug_file_name = debug_file.clone();
        let pdb = debug_file.find(".pdb").unwrap();
        let (symbol_name, _) = debug_file.split_at(pdb);
        let symbol_file = format!("{}.sym", symbol_name);
        let this_url = format!("{}/{}/{}/{}", url, debug_file, debug_id, symbol_file);

        let mut symbol_path = PathBuf::new();
        // TODO get from config
        symbol_path.push("testdata/symbols");

        let mut full_symbol_path = symbol_path.clone();
        full_symbol_path.push(&debug_file);
        full_symbol_path.push(&debug_id);
        full_symbol_path.push(&symbol_file);

        let stack_map_copy = stack_map.clone();

        let _ = create_dir_all(&full_symbol_path.parent().unwrap()).unwrap();

        let supplier = SimpleSymbolSupplier::new(vec!(symbol_path.clone()));

        // TODO most of the time is probably spent waiting on I/O, maybe async would be more appropriate?
        // TODO decide min/max possible threads, possibly based on number of cores?
        handles.push(thread::spawn(move || {

            let symbolizer = Symbolizer::new(supplier);
            // FIXME use Arc<Mutex<File>> to prevent concurrent writes
            // TODO only write contents if server version newer
            if !&full_symbol_path.exists() {

                let mut body = String::new();
                let c = Client::new();
                let mut res = c.get(&this_url).send().unwrap();
                let _ = res.read_to_string(&mut body);

                let mut f = File::create(&full_symbol_path).unwrap();
                let _ = f.write_all(body.as_bytes());
            }

            debug!("symbol_path: {:?}", &symbol_path);
            // FIXME this is having problems with CRLF

            let mut symbols = vec!();
            for stacks in stack_map_copy.get(&counter) {
                for address in stacks {
                    debug!("attempt to symbolicate: {} for: {:?}", *address, &symbol_path);
                    match symbolizer.get_symbol_at_address(&debug_file_name, &debug_id, *address) {
                        Some(x) => symbols.push(x),
                        // return the address rather than function name if symbol not found
                        None => symbols.push(format!("0x{:x}", address)),
                    }
                }
            }
            debug!("debug_file_name: {:?}, symbols: {:?}", debug_file_name, symbols);

            (debug_file_name, symbols)
        }));

        counter += 1;
    }

    let mut result = SymbolResponse {
        symbolicatedStacks: vec!(),
        knownModules: vec!(),
    };

    for handle in handles {
        let (debug_file_name, symbols) = handle.join().unwrap();

        for symbol in symbols {
            result.symbolicatedStacks.push(
                vec!(format!("{} (in {}))", symbol, debug_file_name))
            );
        }
        result.knownModules.push(true);
    }

    json::encode(&result).unwrap()
}

/**
  * Returns individual values from the configuration file.
  */
fn get_config(value_name: &str) -> String {
    // TODO move to actual file, static str for the moment
    let toml: &'static str = r#"
        [symbol_urls]
        public = "https://s3-us-west-2.amazonaws.com/org.mozilla.crash-stats.symbols-public/v1"
    "#;
    // TODO support multiple URLs
    let value: toml::Value = toml.parse().unwrap();

    value.lookup(value_name).unwrap().as_str().unwrap().to_string()
}
<|endoftext|>


<commit_before>extern crate mio;

use std::io::Write;
use std::io::Read;
use std::net::TcpListener;
use std::thread;
use std::sync::{Arc, Mutex};

mod server;
mod broker;
mod message;

struct TcpClient<'a> {
    stream: &'a mut std::net::TcpStream,
}

impl<'a> TcpClient<'a> {
    fn new(stream: &'a mut std::net::TcpStream) -> Self {
        TcpClient { stream : stream }
    }
}

impl<'a> server::Client for TcpClient<'a> {
    fn send(&mut self, bytes: &[u8]) {
        let _ = self.stream.write(bytes);
    }
}

fn main() {
    thread::spawn(miomain);

    let server = Arc::new(Mutex::new(server::Server::new()));
    let listener = TcpListener::bind(("0.0.0.0", 1883)).unwrap();

    for byte_stream in listener.incoming() {
        let server = server.clone();
        let mut mqtt_stream = server::Stream::new();

        thread::spawn(move || {
            let mut byte_stream = byte_stream.unwrap();
            loop {
                let read_result = byte_stream.read(mqtt_stream.buffer());
                //the client must be created here due to scoping issues with byte_stream
                let mut client = TcpClient::new(&mut byte_stream);

                match read_result {
                    Ok(length) => {
                        let mut server = server.lock().unwrap();
                        mqtt_stream.handle_messages(length, &mut server, &mut client);
                    },
                    _ => panic!("Error reading bytes from stream"),
                }
            }
        });
    }
}

const MQTT_SERVER: mio::Token = mio::Token(0);

fn miomain() {
    let address = "0.0.0.0:1884".parse().unwrap();
    let listener = mio::tcp::TcpListener::bind(&address).unwrap();
    let mut event_loop = mio::EventLoop::new().unwrap();
    event_loop.register(&listener, MQTT_SERVER);
    println!("Running mio server");
    event_loop.run(&mut MioHandler::new(listener));
}

struct MioHandler {
    listener: mio::tcp::TcpListener,
    connections: mio::util::Slab<Connection>,
}

impl MioHandler {
    fn new(listener: mio::tcp::TcpListener) -> Self {
        let slab = mio::util::Slab::new_starting_at(mio::Token(1), 1024 * 32);

        MioHandler {
            listener: listener,
            connections: slab,
        }
    }
}

struct Connection {
    socket: mio::tcp::TcpStream,
    token: mio::Token,
}

impl Connection {
    fn new(socket: mio::tcp::TcpStream, token: mio::Token) -> Self {
        Connection { socket: socket, token: token }
    }

    fn ready(&mut self, event_loop: &mut mio::EventLoop<MioHandler>, events: mio::EventSet) {
        println!("Connection ready!");
        assert!(events.is_readable());
        let mut buffer : Vec<u8> = vec![0; 1024];
        let res = self.socket.read(&mut buffer[..]);
        match res {
            Ok(length) => {
                println!("Read these bytes: {:?}", &buffer[0..length]);
            }
            _ => {
                println!("Could not read from client socket");
            }
        }
    }
}

impl mio::Handler for MioHandler {
    type Timeout = ();
    type Message = ();

    fn ready(&mut self,
             event_loop: &mut mio::EventLoop<MioHandler>,
             token: mio::Token,
             events: mio::EventSet) {
        match token {
            MQTT_SERVER => {
                assert!(events.is_readable());

                println!("Server socket is ready to accept a connetion");
                match self.listener.accept() {
                    Ok(Some(socket)) => {
                        println!("New mio client connection");
                        let token = self.connections
                            .insert_with(|token| Connection::new(socket, token))
                            .unwrap();
                        event_loop.register_opt(
                            &self.connections[token].socket,
                            token,
                            mio::EventSet::readable(),
                            mio::PollOpt::edge()).unwrap();
                    }
                    Ok(None) => {
                        println!("The server socket wasn't actually ready");
                    }
                    Err(e) => {
                        println!("listener.accept errored: {}", e);
                        event_loop.shutdown();
                    }
                }
            }
            _ => {
                self.connections[token].ready(event_loop, events);
            }
        }
    }
}
<commit_msg>Cucumber tests passing with mio<commit_after>extern crate mio;

use std::io::Write;
use std::io::Read;
use std::net::TcpListener;
use std::thread;
use std::sync::{Arc, Mutex};

mod server;
mod broker;
mod message;

struct TcpClient<'a> {
    stream: &'a mut std::net::TcpStream,
}

impl<'a> TcpClient<'a> {
    fn new(stream: &'a mut std::net::TcpStream) -> Self {
        TcpClient { stream : stream }
    }
}

impl<'a> server::Client for TcpClient<'a> {
    fn send(&mut self, bytes: &[u8]) {
        let _ = self.stream.write(bytes);
    }
}

fn main() {
    thread::spawn(miomain);

    let server = Arc::new(Mutex::new(server::Server::new()));
    let listener = TcpListener::bind(("0.0.0.0", 1884)).unwrap();

    for byte_stream in listener.incoming() {
        let server = server.clone();
        let mut mqtt_stream = server::Stream::new();

        thread::spawn(move || {
            let mut byte_stream = byte_stream.unwrap();
            loop {
                let read_result = byte_stream.read(mqtt_stream.buffer());
                //the client must be created here due to scoping issues with byte_stream
                let mut client = TcpClient::new(&mut byte_stream);

                match read_result {
                    Ok(length) => {
                        let mut server = server.lock().unwrap();
                        mqtt_stream.handle_messages(length, &mut server, &mut client);
                    },
                    _ => panic!("Error reading bytes from stream"),
                }
            }
        });
    }
}

const MQTT_SERVER_TOKEN: mio::Token = mio::Token(0);

fn miomain() {
    let address = "0.0.0.0:1883".parse().unwrap();
    let listener = mio::tcp::TcpListener::bind(&address).unwrap();
    let mut event_loop = mio::EventLoop::new().unwrap();
    event_loop.register(&listener, MQTT_SERVER_TOKEN);
    println!("Running mio server");
    event_loop.run(&mut MioHandler::new(listener));
}

struct MioHandler {
    listener: mio::tcp::TcpListener,
    connections: mio::util::Slab<Connection>,
    server: server::Server,
}

impl MioHandler {
    fn new(listener: mio::tcp::TcpListener) -> Self {
        let slab = mio::util::Slab::new_starting_at(mio::Token(1), 1024 * 32);

        MioHandler {
            listener: listener,
            connections: slab,
            server: server::Server::new(),
        }
    }
}

struct Connection {
    socket: mio::tcp::TcpStream,
    token: mio::Token,
    stream: server::Stream,
}

struct TcpClient2<'a> {
    stream: &'a mut mio::tcp::TcpStream,
}

impl<'a> TcpClient2<'a> {
    fn new(stream: &'a mut mio::tcp::TcpStream) -> Self {
        TcpClient2 { stream : stream }
    }
}

impl<'a> server::Client for TcpClient2<'a> {
    fn send(&mut self, bytes: &[u8]) {
        let _ = self.stream.write(bytes);
    }
}

impl Connection {
    fn new(socket: mio::tcp::TcpStream, token: mio::Token) -> Self {
        Connection { socket: socket, token: token, stream: server::Stream::new() }
    }

    fn ready(&mut self, event_loop: &mut mio::EventLoop<MioHandler>, events: mio::EventSet, server: &mut server::Server) {
        println!("Connection ready!");
        assert!(events.is_readable());
        let read_result = self.socket.read(self.stream.buffer());
        let mut client = TcpClient2::new(&mut self.socket);

        match read_result {
            Ok(length) => {
                self.stream.handle_messages(length, server, &mut client);
            },
            _ => panic!("Error reading bytes from stream"),
        }
    }
}

impl mio::Handler for MioHandler {
    type Timeout = ();
    type Message = ();

    fn ready(&mut self,
             event_loop: &mut mio::EventLoop<MioHandler>,
             token: mio::Token,
             events: mio::EventSet) {
        match token {
            MQTT_SERVER_TOKEN => {
                assert!(events.is_readable());

                println!("Server socket is ready to accept a connetion");
                match self.listener.accept() {
                    Ok(Some(socket)) => {
                        println!("New mio client connection");
                        let token = self.connections
                            .insert_with(|token| Connection::new(socket, token))
                            .unwrap();
                        event_loop.register_opt(
                            &self.connections[token].socket,
                            token,
                            mio::EventSet::readable(),
                            mio::PollOpt::edge()).unwrap();
                    }
                    Ok(None) => {
                        println!("The server socket wasn't actually ready");
                    }
                    Err(e) => {
                        println!("listener.accept errored: {}", e);
                        event_loop.shutdown();
                    }
                }
            }
            _ => {
                self.connections[token].ready(event_loop, events, &mut self.server);
            }
        }
    }
}
<|endoftext|>