<commit_before>extern crate mmap;
extern crate time;
extern crate timely;
extern crate getopts;

extern crate graph_map;
extern crate graph_layout;
extern crate radix_sort;

use timely::progress::timestamp::RootTimestamp;
use timely::progress::nested::Summary::Local;
use timely::dataflow::*;
use timely::dataflow::operators::*;
use timely::dataflow::channels::pact::{Pipeline, Exchange};
use timely::drain::DrainExt;

use graph_layout::layout::*;
use graph_layout::compression::*;

use graph_map::graph_map::GraphMMap;

use radix_sort::*;

fn main () {

    /*
        The plan here is to partition the rows of the matrix into #processes parts, and the columns
        into #threads parts. This could be generally #row_parts and #col_parts, but that requires
        more thinking about things.
    */

    let threads: u32 = ::std::env::args().nth(1).unwrap().parse().unwrap();
    let filename = ::std::env::args().nth(2).unwrap();
    let nodes: u32 = ::std::env::args().nth(3).unwrap().parse().unwrap();

    timely::execute_from_args(::std::env::args().skip(4), move |root| {

        let index = root.index() as u32;
        let peers = root.peers() as u32;
        let processes = (peers / threads) as u32;
        let dst_index = index % threads;

        let start = time::precise_time_s();
        let mut going = start;

        let mut counter = 0;
        let mut sorter = RadixSorter::new();
        let hilbert = Hilbert::new();
        let mut decoder = BytewiseCached::new();
        let mut compressed = vec![];

        // let nodes: u32 = 3_563_602_788 + 1;

        let local_srcs = (nodes / processes) + 1;
        let local_dsts = (nodes / threads) + 1;

        // degree is ignored at the moment.
        // mostly just a pain to put together.
        let deg = vec![1.0f32; local_srcs as usize];
        let mut src = vec![1.0f32; local_srcs as usize];
        let mut dst = vec![0.0f32; local_dsts as usize];

        let mut input = root.scoped(|builder| {

            let (input, edges) = builder.new_input::<(u32, u32)>();
            let (cycle, ranks) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

            let ranks = edges.binary_notify(&ranks,
                                Exchange::new(|x: &(u32,u32)| ((x.0 % processes) * threads + (x.1 % threads)) as u64),
                                Pipeline,   // data are partitioned as part of the broadcast step.
                                "hilbrank",
                                vec![RootTimestamp::new(0)],
                                move |input1, input2, output, notificator| {

                // receive incoming edges (should only be iter 0)
                while let Some((_index, data)) = input1.next() {
                    counter += data.len();
                    for &(src,dst) in data.iter() {
                        let shrunk = (src / processes, dst / threads);
                        sorter.push(hilbert.entangle(shrunk), &|&x| x);
                    }

                    // should be 1GB per worker
                    if counter > (1 << 27) {
                        let mut results = sorter.finish(&|&x| x);
                        let mut compressor = Compressor::with_capacity(counter);
                        for vec in results.iter_mut() {
                            for elem in vec.drain_temp() {
                                compressor.push(elem);
                            }
                        }
                        sorter.recycle(results);
                        compressed.push(compressor.done());
                        counter = 0;
                    }
                }

                if counter > 0 {
                    // round out any remaining edges
                    let mut results = sorter.finish(&|&x| x);
                    let mut compressor = Compressor::with_capacity(counter);
                    for vec in results.iter_mut() {
                        for elem in vec.drain_temp() {
                            compressor.push(elem);
                        }
                    }
                    compressed.push(compressor.done());
                    counter = 0;
                }

                // all inputs received for iter, commence multiplication
                while let Some((iter, _)) = notificator.next() {

                    let now = time::now();

                    if index == 0 { println!("{}:{}:{}.{} starting iteration {}", now.tm_hour, now.tm_min, now.tm_sec, now.tm_nsec, iter.inner); }

                    // if the very first iteration, prepare some stuff.
                    if iter.inner == 0 {
                        // now we would merge the compressed representations, if we want to.
                        // let's measure both doing this and not doing it. :)

                    }

                    // record some timings in order to estimate per-iteration times
                    // if iter.inner == 0  && index == 0 { println!("src: {}, dst: {}, edges: {}", src.len(), rev.len(), trn.len()); }
                    if iter.inner == 10 && index == 0 { going = time::precise_time_s(); }
                    if iter.inner == 20 && index == 0 { println!("average: {}", (time::precise_time_s() - going) / 10.0 ); }

                    // prepare src for transmitting to destinations
                    for s in 0..src.len() { src[s] = 0.15 + 0.85 * src[s] / deg[s] as f32; }
                    for d in 0..dst.len() { dst[d] = 0.0; }

                    // do the multiplication for each compressed sequence
                    for sequence in &compressed {
                        for element in sequence.decompress() {
                            let (s,d) = decoder.detangle(element);
                            dst[d as usize] += src[s as usize];
                        }
                    }

                    for s in 0..src.len() { src[s] = 0.0; }

                    // now we need to send some stuff to other threads.
                    // TODO : This could be done as blocks, as they mostly go to the same place.
                    let mut session = output.session(&iter);
                    for (index, &value) in dst.iter().enumerate() {
                        if value != 0.0 {
                            session.give((index as u32 * threads + dst_index, value));
                        }
                    }
                }

                // receive data from threads, accumulate in src
                while let Some((iter, data)) = input2.next() {
                    notificator.notify_at(&iter);
                    for &(node, rank) in data.iter() {
                        src[(node / processes) as usize] += rank;
                    }
                }
            });

            let mut aggregates = vec![0.0f32; (nodes / peers) as usize];
            // want to do aggregation so that aggregates are local to the folks who will need them.
            // that is, (node, rank) goes to some worker on machine#: name % processes
            let aggregated = ranks.unary_notify(
                Exchange::new(move |&(n,_): &(u32,f32)| (((n % processes) * threads) + ((n / processes) % threads)) as u64),
                "aggregation",
                vec![],
                move |input, output, notificator | {

                while let Some((time, data)) = input.next() {
                    notificator.notify_at(time);
                    for &(node, rank) in data.iter() {
                        aggregates[(node / peers) as usize] += rank;
                    }
                }

                while let Some((time, _)) = notificator.next() {
                    let mut session = output.session(&time);
                    for (node, &rank) in aggregates.iter().enumerate() {
                        if rank != 0.0 {
                            session.give((node as u32 * peers, rank));
                        }
                    }
                }
            });

            // data are now aggregated, and only need to be broadcast out to the intended recipients
            // ideally, none of this actually ends up hitting the network. also, it seems like it
            // shouldn't even require looking at `node`, on account of already putting all the data
            // on the right processes. We should be able to use (index / threads) * threads, I think.
            let mut exchanged = aggregated.exchange(move |&(node, _)| ((node % processes) * threads) as u64);
            for i in 1..threads {
                exchanged = aggregated.exchange(move |&(node, _)| (((node % processes) * threads) + i) as u64)
                                      .concat(&exchanged);
            }

            exchanged.connect_loop(cycle);

            input
        });

        // introduce edges into the computation;
        // allow mmaped file to drop
        {
            let graph = GraphMMap::new(&filename);
            for node in 0..graph.nodes() {
                if node as u32 % peers == index {
                    for dst in graph.edges(node) {
                        input.send((node as u32, *dst as u32));
                    }
                }
            }
        }
        input.close();
        while root.step() { }

        if index == 0 { println!("elapsed: {}", time::precise_time_s() - start); }
    });

}
<commit_msg>lolwoops<commit_after>extern crate mmap;
extern crate time;
extern crate timely;
extern crate getopts;

extern crate graph_map;
extern crate graph_layout;
extern crate radix_sort;

use timely::progress::timestamp::RootTimestamp;
use timely::progress::nested::Summary::Local;
use timely::dataflow::*;
use timely::dataflow::operators::*;
use timely::dataflow::channels::pact::{Pipeline, Exchange};
use timely::drain::DrainExt;

use graph_layout::layout::*;
use graph_layout::compression::*;

use graph_map::graph_map::GraphMMap;

use radix_sort::*;

fn main () {

    /*
        The plan here is to partition the rows of the matrix into #processes parts, and the columns
        into #threads parts. This could be generally #row_parts and #col_parts, but that requires
        more thinking about things.
    */

    let threads: u32 = ::std::env::args().nth(1).unwrap().parse().unwrap();
    let filename = ::std::env::args().nth(2).unwrap();
    let nodes: u32 = ::std::env::args().nth(3).unwrap().parse().unwrap();

    timely::execute_from_args(::std::env::args().skip(4), move |root| {

        let index = root.index() as u32;
        let peers = root.peers() as u32;
        let processes = (peers / threads) as u32;
        let dst_index = index % threads;

        let start = time::precise_time_s();
        let mut going = start;

        let mut counter = 0;
        let mut sorter = RadixSorter::new();
        let hilbert = Hilbert::new();
        let mut decoder = BytewiseCached::new();
        let mut compressed = vec![];

        // let nodes: u32 = 3_563_602_788 + 1;

        let local_srcs = (nodes / processes) + 1;
        let local_dsts = (nodes / threads) + 1;

        // degree is ignored at the moment.
        // mostly just a pain to put together.
        let deg = vec![1.0f32; local_srcs as usize];
        let mut src = vec![1.0f32; local_srcs as usize];
        let mut dst = vec![0.0f32; local_dsts as usize];

        let mut input = root.scoped(|builder| {

            let (input, edges) = builder.new_input::<(u32, u32)>();
            let (cycle, ranks) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

            let ranks = edges.binary_notify(&ranks,
                                Exchange::new(move |x: &(u32,u32)| ((x.0 % processes) * threads + (x.1 % threads)) as u64),
                                Pipeline,   // data are partitioned as part of the broadcast step.
                                "hilbrank",
                                vec![RootTimestamp::new(0)],
                                move |input1, input2, output, notificator| {

                // receive incoming edges (should only be iter 0)
                while let Some((_index, data)) = input1.next() {
                    counter += data.len();
                    for &(src,dst) in data.iter() {
                        let shrunk = (src / processes, dst / threads);
                        sorter.push(hilbert.entangle(shrunk), &|&x| x);
                    }

                    // should be 1GB per worker
                    if counter > (1 << 27) {
                        let mut results = sorter.finish(&|&x| x);
                        let mut compressor = Compressor::with_capacity(counter);
                        for vec in results.iter_mut() {
                            for elem in vec.drain_temp() {
                                compressor.push(elem);
                            }
                        }
                        sorter.recycle(results);
                        compressed.push(compressor.done());
                        counter = 0;
                    }
                }

                if counter > 0 {
                    // round out any remaining edges
                    let mut results = sorter.finish(&|&x| x);
                    let mut compressor = Compressor::with_capacity(counter);
                    for vec in results.iter_mut() {
                        for elem in vec.drain_temp() {
                            compressor.push(elem);
                        }
                    }
                    compressed.push(compressor.done());
                    counter = 0;
                }

                // all inputs received for iter, commence multiplication
                while let Some((iter, _)) = notificator.next() {

                    let now = time::now();

                    if index == 0 { println!("{}:{}:{}.{} starting iteration {}", now.tm_hour, now.tm_min, now.tm_sec, now.tm_nsec, iter.inner); }

                    // if the very first iteration, prepare some stuff.
                    if iter.inner == 0 {
                        // now we would merge the compressed representations, if we want to.
                        // let's measure both doing this and not doing it. :)

                    }

                    // record some timings in order to estimate per-iteration times
                    // if iter.inner == 0  && index == 0 { println!("src: {}, dst: {}, edges: {}", src.len(), rev.len(), trn.len()); }
                    if iter.inner == 10 && index == 0 { going = time::precise_time_s(); }
                    if iter.inner == 20 && index == 0 { println!("average: {}", (time::precise_time_s() - going) / 10.0 ); }

                    // prepare src for transmitting to destinations
                    for s in 0..src.len() { src[s] = 0.15 + 0.85 * src[s] / deg[s] as f32; }
                    for d in 0..dst.len() { dst[d] = 0.0; }

                    // do the multiplication for each compressed sequence
                    for sequence in &compressed {
                        for element in sequence.decompress() {
                            let (s,d) = decoder.detangle(element);
                            dst[d as usize] += src[s as usize];
                        }
                    }

                    for s in 0..src.len() { src[s] = 0.0; }

                    // now we need to send some stuff to other threads.
                    // TODO : This could be done as blocks, as they mostly go to the same place.
                    let mut session = output.session(&iter);
                    for (index, &value) in dst.iter().enumerate() {
                        if value != 0.0 {
                            session.give((index as u32 * threads + dst_index, value));
                        }
                    }
                }

                // receive data from threads, accumulate in src
                while let Some((iter, data)) = input2.next() {
                    notificator.notify_at(&iter);
                    for &(node, rank) in data.iter() {
                        src[(node / processes) as usize] += rank;
                    }
                }
            });

            let mut aggregates = vec![0.0f32; (nodes / peers) as usize];
            // want to do aggregation so that aggregates are local to the folks who will need them.
            // that is, (node, rank) goes to some worker on machine#: name % processes
            let aggregated = ranks.unary_notify(
                Exchange::new(move |&(n,_): &(u32,f32)| (((n % processes) * threads) + ((n / processes) % threads)) as u64),
                "aggregation",
                vec![],
                move |input, output, notificator | {

                while let Some((time, data)) = input.next() {
                    notificator.notify_at(time);
                    for &(node, rank) in data.iter() {
                        aggregates[(node / peers) as usize] += rank;
                    }
                }

                while let Some((time, _)) = notificator.next() {
                    let mut session = output.session(&time);
                    for (node, &rank) in aggregates.iter().enumerate() {
                        if rank != 0.0 {
                            session.give((node as u32 * peers, rank));
                        }
                    }
                }
            });

            // data are now aggregated, and only need to be broadcast out to the intended recipients
            // ideally, none of this actually ends up hitting the network. also, it seems like it
            // shouldn't even require looking at `node`, on account of already putting all the data
            // on the right processes. We should be able to use (index / threads) * threads, I think.
            let mut exchanged = aggregated.exchange(move |&(node, _)| ((node % processes) * threads) as u64);
            for i in 1..threads {
                exchanged = aggregated.exchange(move |&(node, _)| (((node % processes) * threads) + i) as u64)
                                      .concat(&exchanged);
            }

            exchanged.connect_loop(cycle);

            input
        });

        // introduce edges into the computation;
        // allow mmaped file to drop
        {
            let graph = GraphMMap::new(&filename);
            for node in 0..graph.nodes() {
                if node as u32 % peers == index {
                    for dst in graph.edges(node) {
                        input.send((node as u32, *dst as u32));
                    }
                }
            }
        }
        input.close();
        while root.step() { }

        if index == 0 { println!("elapsed: {}", time::precise_time_s() - start); }
    });

}
<|endoftext|>


<commit_before>#![feature(plugin)]
#![plugin(clippy)]

extern crate rustc_serialize;
extern crate docopt;
extern crate rusqlite;
extern crate walkdir;
extern crate regex;
#[macro_use(log, info, error)]
extern crate log;
extern crate stdio_logger;

use docopt::Docopt;
use rusqlite::Connection;
use std::iter::Iterator;
use walkdir::{DirEntry, WalkDir, WalkDirIterator};
use std::path::{Path, Component};
use std::fs::File;
use std::io;
use std::io::prelude::*;
use log::{Log, LogLevel};
use stdio_logger::Logger;

const USAGE: &'static str = "
MBTiles utils.

Usage:
    mbutiles <command> [options] <input> \
                             [<output>]
    mbutiles -h | --help
    mbutiles --version

Options:
  -h --help                   Show this help message and exit.
  --verbose                   Show log info.
  --version                   Show version.
  --scheme=<scheme>           Tiling scheme of the tiles. Default is \"xyz\" (z/x/y), other options are \"tms\" which is also z/x/y but uses a flipped y coordinate, and \"wms\" which replicates the MapServer WMS TileCache directory structure \"z/000/000/x/000/000/y.png\". [default: xyz]
  --image-format=<format>     The format of the image tiles, either png, jpg, webp or pbf.
  --grid-callback=<callback>  Option to control JSONP callback for UTFGrid tiles. If grids are not used as JSONP, you can remove callbacks specifying --grid_callback=\"\".

 Commands:
    import
    export
    metadata
";

#[derive(RustcDecodable, Debug)]
enum Command {
    Import,
    Export,
    Metadata,
}

#[derive(RustcDecodable, Debug)]
enum Scheme {
    Xyz,
    Tms,
    Wms,
}

#[derive(RustcDecodable, Debug, Clone, Copy)]
enum ImageFormat {
    Png,
    Jpg,
    Webp,
    Pbf,
}

#[derive(RustcDecodable, Debug)]
struct Args {
    arg_command: Command,
    flag_verbose: bool,
    flag_scheme: Scheme,
    flag_image_format: Option<ImageFormat>,
    flag_grid_callback: Option<String>,
    arg_input: String,
    arg_output: Option<String>,
}

fn main() {
    let args: Args = Docopt::new(USAGE)
                         .and_then(|d| d.decode())
                         .unwrap_or_else(|e| e.exit());
    stdio_logger::init(if args.flag_verbose {LogLevel::Info} else {LogLevel::Error})
        .expect("Could not initialize logging");
    info!("{:?}", args);
    match args.arg_command {
        Command::Import => {
            // import tiles dir into mbtiles
            let input = args.arg_input.clone();
            let output = args.arg_output.unwrap_or_else(
                || format!("{}.mbtiles", input));
            import(&Path::new(&args.arg_input), &Path::new(&output),
                args.flag_scheme, args.flag_image_format, args.flag_grid_callback)
        },
        Command::Export =>
            // export mbtiles to a dir
            export(args.arg_input, args.arg_output,
                args.flag_scheme, args.flag_image_format, args.flag_grid_callback),
        Command::Metadata =>
            // dumps metadata
            metadata(args.arg_input, args.arg_output,
                args.flag_scheme, args.flag_image_format, args.flag_grid_callback),
    }
}

fn mbtiles_connect(mbtiles_file: &Path) -> Connection {
    Connection::open(mbtiles_file).unwrap()
}

fn optimize_connection(connection: &Connection) {
    if let Err(err) = connection.execute_batch("
        PRAGMA synchronous=0;
        PRAGMA locking_mode=EXCLUSIVE;
        PRAGMA journal_mode=DELETE;
        ") {
        error!("Cannot execute sqlite optimization query {:?}", err);
    }
}

struct Tile {
    zoom_level: Option<i32>,
    tile_column: Option<i32>,
    tile_row: Option<i32>,
    tile_data: Option<Vec<u8>>,
}

fn mbtiles_setup(connection: &Connection) {
    connection.execute_batch("
    CREATE TABLE tiles (
            zoom_level INTEGER,
            tile_column INTEGER,
            tile_row INTEGER,
            tile_data BLOB);
    CREATE TABLE metadata
        (name TEXT, value TEXT);
    CREATE TABLE grids (zoom_level INTEGER, tile_column INTEGER,
        tile_row INTEGER, grid BLOB);
    CREATE TABLE grid_data (zoom_level INTEGER, tile_column
        INTEGER, tile_row INTEGER, key_name TEXT, key_json TEXT);
    CREATE UNIQUE INDEX name ON metadata (name);
    CREATE UNIQUE INDEX tile_index ON tiles
        (zoom_level, tile_column, tile_row);
    ");
}

fn is_hidden(entry: &DirEntry) -> bool {
    entry.file_name()
         .to_str()
         .map(|s| s.starts_with("."))
         .unwrap_or(false)
}

fn get_extension(image_format: ImageFormat) -> String {
    match image_format {
        ImageFormat::Jpg => "jpg".to_owned(),
        ImageFormat::Pbf => "pbf".to_owned(),
        ImageFormat::Png => "png".to_owned(),
        ImageFormat::Webp => "webp".to_owned(),
    }
}

fn parse_component(component: Component, parse_file: Option<ImageFormat>) -> Option<u32> {
    if let Component::Normal(zoom_dir) = component {
        zoom_dir.to_str()
            .ok_or("no component".to_owned())
            .and_then(|s| {
                if let Some(image_format) = parse_file {
                    let parts: Vec<&str> = s.split('.').collect();
                    let filtered_extension = get_extension(image_format);
                    if parts[1] == filtered_extension {
                        parts[0].parse::<u32>().map_err(|err| err.to_string())
                    } else {
                        Err(format!("The filtered extention {} is different \
than the path\'s extention {}", filtered_extension, parts[1]).to_owned())
                    }
                } else {
                    s.parse::<u32>().map_err(|err| err.to_string())
                }
            })
            .map_err(|err| error!("{:?}", err))
            .ok()
    } else {
        error!("Can't read path component {:?}", component);
        None
    }
}

fn import(input: &Path,
          output: &Path,
          flag_scheme: Scheme,
          flag_image_format: Option<ImageFormat>,
          flag_grid_callback: Option<String>) {
    let input_path = Path::new(&input);
    if input_path.is_dir() {
        let connection = mbtiles_connect(output);
        optimize_connection(&connection);
        mbtiles_setup(&connection);
        let base_components_length = input_path.components().count();
        let dir_walker = WalkDir::new(input_path)
            .follow_links(true)
            .min_depth(1)
            .max_depth(3)
            .into_iter()
            .filter_entry(|e| !is_hidden(e));
        for entry in dir_walker {
            if let Ok(entry) = entry {
                let entry_path = entry.path();
                if entry_path.is_file() {
                    let end_comp : Vec<Component> = entry_path
                        .components()
                        .skip(base_components_length)
                        .collect();
                    if end_comp.len() == 3 {
                        if let Some(zoom) = parse_component(end_comp[0], None) {
                            if let Some(row) = parse_component(end_comp[1], None) {
                                if let Some(col) = parse_component(end_comp[2], flag_image_format) {
                                    info!("Zoom: {}, Row: {}, Col {}", zoom, row, col);
                                    insert_image_sqlite(entry_path, zoom, col, row, &connection);
                                }
                            }
                        }
                    }
                }
                info!("{}", entry.path().display());

            }
        }
    } else {
        error!("Can only import from a directory")
    }
}

fn insert_image_sqlite(image_path: &Path, zoom: u32, column: u32, row: u32, connection: &Connection) {
    match File::open(image_path) {
        Ok(mut image_file) =>  {
            let mut buffer = Vec::new();
            if image_file.read_to_end(&mut buffer).is_ok() {
                if let Err(err) = connection.execute("insert into tiles (zoom_level,
                            tile_column, tile_row, tile_data) values
                            ($1, $2, $3, $4);",
                            &[&(zoom as i64), &(column as i64), &(row as i64), &buffer]) {
                                error!("Can't insert {:?}, {}", image_path, err);
                }
            } else {
                error!("Can't read file {:?}", image_path);
            }
        },
        Err(err) => {
            error!("Can't open {:?} =>  {:?}", image_path, err);
        }
    }
}

fn export(input: String,
          output: Option<String>,
          flag_scheme: Scheme,
          flag_image_format: Option<ImageFormat>,
          flag_grid_callback: Option<String>) {
    let input_path = Path::new(&input);
    if input_path.is_file() {
    } else {
        error!("Can only export from a file")
    }
}

fn metadata(input: String,
            output: Option<String>,
            flag_scheme: Scheme,
            flag_image_format: Option<ImageFormat>,
            flag_grid_callback: Option<String>) {
    let input_path = Path::new(&input);
    if input_path.is_file() {
    } else {
        error!("Can only dumps from a file")
    }
}
<commit_msg>apply rustfmt formatting, fix the usage string<commit_after>#![feature(plugin)]
#![plugin(clippy)]

extern crate rustc_serialize;
extern crate docopt;
extern crate rusqlite;
extern crate walkdir;
extern crate regex;
#[macro_use(log, info, error)]
extern crate log;
extern crate stdio_logger;

use docopt::Docopt;
use rusqlite::Connection;
use std::iter::Iterator;
use walkdir::{DirEntry, WalkDir, WalkDirIterator};
use std::path::{Path, Component};
use std::fs::File;
use std::io;
use std::io::prelude::*;
use log::{Log, LogLevel};
use stdio_logger::Logger;

const USAGE: &'static str = "
MBTiles utils.

Usage:
    mbutiles <command> [options] <input> \
                             [<output>]
    mbutiles -h | --help
    mbutiles --version

Options:
  -h --help                   Show this help message and exit.
  --verbose                   Show log info.
  --version                   Show version.
  --scheme=<scheme>           Tiling scheme of the tiles. Default is \"xyz\" (z/x/y),\
 other options are \"tms\" which is also z/x/y but uses a flipped y coordinate,\
 and \"wms\" which replicates the MapServer WMS TileCache directory structure\
 \"z/000/000/x/000/000/y.png\". [default: xyz]
  --image-format=<format>     The format of the image tiles, either png, jpg, webp or pbf.
  --grid-callback=<callback>  Option to control JSONP callback for UTFGrid tiles.\
 If grids are not used as JSONP, you can remove callbacks specifying --grid_callback=\"\".

 Commands:
    import
    export
    metadata
";

#[derive(RustcDecodable, Debug)]
enum Command {
    Import,
    Export,
    Metadata,
}

#[derive(RustcDecodable, Debug)]
enum Scheme {
    Xyz,
    Tms,
    Wms,
}

#[derive(RustcDecodable, Debug, Clone, Copy)]
enum ImageFormat {
    Png,
    Jpg,
    Webp,
    Pbf,
}

#[derive(RustcDecodable, Debug)]
struct Args {
    arg_command: Command,
    flag_verbose: bool,
    flag_scheme: Scheme,
    flag_image_format: Option<ImageFormat>,
    flag_grid_callback: Option<String>,
    arg_input: String,
    arg_output: Option<String>,
}

fn main() {
    let args: Args = Docopt::new(USAGE)
                         .and_then(|d| d.decode())
                         .unwrap_or_else(|e| e.exit());
    stdio_logger::init(if args.flag_verbose {
        LogLevel::Info
    } else {
        LogLevel::Error
    })
        .expect("Could not initialize logging");
    info!("{:?}", args);
    match args.arg_command {
        Command::Import => {
            // import tiles dir into mbtiles
            let input = args.arg_input.clone();
            let output = args.arg_output.unwrap_or_else(
                || format!("{}.mbtiles", input));
            import(&Path::new(&args.arg_input), &Path::new(&output),
                args.flag_scheme, args.flag_image_format, args.flag_grid_callback)
        },
        Command::Export =>
            // export mbtiles to a dir
            export(args.arg_input, args.arg_output,
                args.flag_scheme, args.flag_image_format, args.flag_grid_callback),
        Command::Metadata =>
            // dumps metadata
            metadata(args.arg_input, args.arg_output,
                args.flag_scheme, args.flag_image_format, args.flag_grid_callback),
    }
}

fn mbtiles_connect(mbtiles_file: &Path) -> Connection {
    Connection::open(mbtiles_file).unwrap()
}

fn optimize_connection(connection: &Connection) {
    if let Err(err) = connection.execute_batch("
        PRAGMA synchronous=0;
        PRAGMA locking_mode=EXCLUSIVE;
        PRAGMA journal_mode=DELETE;
        ") {
        error!("Cannot execute sqlite optimization query {:?}", err);
    }
}

struct Tile {
    zoom_level: Option<i32>,
    tile_column: Option<i32>,
    tile_row: Option<i32>,
    tile_data: Option<Vec<u8>>,
}

fn mbtiles_setup(connection: &Connection) {
    connection.execute_batch("
    CREATE TABLE tiles (
            zoom_level INTEGER,
            tile_column INTEGER,
            tile_row INTEGER,
            tile_data BLOB);
    CREATE TABLE metadata
        (name TEXT, value TEXT);
    CREATE TABLE grids (zoom_level INTEGER, tile_column INTEGER,
        tile_row INTEGER, grid BLOB);
    CREATE TABLE grid_data (zoom_level INTEGER, tile_column
        INTEGER, tile_row INTEGER, key_name TEXT, key_json TEXT);
    CREATE UNIQUE INDEX name ON metadata (name);
    CREATE UNIQUE INDEX tile_index ON tiles
        (zoom_level, tile_column, tile_row);
    ");
}

fn is_hidden(entry: &DirEntry) -> bool {
    entry.file_name()
         .to_str()
         .map(|s| s.starts_with("."))
         .unwrap_or(false)
}

fn get_extension(image_format: ImageFormat) -> String {
    match image_format {
        ImageFormat::Jpg => "jpg".to_owned(),
        ImageFormat::Pbf => "pbf".to_owned(),
        ImageFormat::Png => "png".to_owned(),
        ImageFormat::Webp => "webp".to_owned(),
    }
}

fn parse_component(component: Component, parse_file: Option<ImageFormat>) -> Option<u32> {
    if let Component::Normal(zoom_dir) = component {
        zoom_dir.to_str()
                .ok_or("no component".to_owned())
                .and_then(|s| {
                    if let Some(image_format) = parse_file {
                        let parts: Vec<&str> = s.split('.').collect();
                        let filtered_extension = get_extension(image_format);
                        if parts[1] == filtered_extension {
                            parts[0].parse::<u32>().map_err(|err| err.to_string())
                        } else {
                            Err(format!("The filtered extention {} is different than the path\'s \
                                         extention {}",
                                        filtered_extension,
                                        parts[1])
                                    .to_owned())
                        }
                    } else {
                        s.parse::<u32>().map_err(|err| err.to_string())
                    }
                })
                .map_err(|err| error!("{:?}", err))
                .ok()
    } else {
        error!("Can't read path component {:?}", component);
        None
    }
}

fn import(input: &Path,
          output: &Path,
          flag_scheme: Scheme,
          flag_image_format: Option<ImageFormat>,
          flag_grid_callback: Option<String>) {
    let input_path = Path::new(&input);
    if input_path.is_dir() {
        let connection = mbtiles_connect(output);
        optimize_connection(&connection);
        mbtiles_setup(&connection);
        let base_components_length = input_path.components().count();
        let dir_walker = WalkDir::new(input_path)
                             .follow_links(true)
                             .min_depth(1)
                             .max_depth(3)
                             .into_iter()
                             .filter_entry(|e| !is_hidden(e));
        for entry in dir_walker {
            if let Ok(entry) = entry {
                let entry_path = entry.path();
                if entry_path.is_file() {
                    let end_comp: Vec<Component> = entry_path.components()
                                                             .skip(base_components_length)
                                                             .collect();
                    if end_comp.len() == 3 {
                        if let Some(zoom) = parse_component(end_comp[0], None) {
                            if let Some(row) = parse_component(end_comp[1], None) {
                                if let Some(col) = parse_component(end_comp[2], flag_image_format) {
                                    info!("Zoom: {}, Row: {}, Col {}", zoom, row, col);
                                    insert_image_sqlite(entry_path, zoom, col, row, &connection);
                                }
                            }
                        }
                    }
                }
                info!("{}", entry.path().display());

            }
        }
    } else {
        error!("Can only import from a directory")
    }
}

fn insert_image_sqlite(image_path: &Path,
                       zoom: u32,
                       column: u32,
                       row: u32,
                       connection: &Connection) {
    match File::open(image_path) {
        Ok(mut image_file) => {
            let mut buffer = Vec::new();
            if image_file.read_to_end(&mut buffer).is_ok() {
                if let Err(err) = connection.execute("insert into tiles (zoom_level,
                            tile_column, tile_row, tile_data) values
                            ($1, $2, $3, $4);",
                                                     &[&(zoom as i64),
                                                       &(column as i64),
                                                       &(row as i64),
                                                       &buffer]) {
                    error!("Can't insert {:?}, {}", image_path, err);
                }
            } else {
                error!("Can't read file {:?}", image_path);
            }
        }
        Err(err) => {
            error!("Can't open {:?} =>  {:?}", image_path, err);
        }
    }
}

fn export(input: String,
          output: Option<String>,
          flag_scheme: Scheme,
          flag_image_format: Option<ImageFormat>,
          flag_grid_callback: Option<String>) {
    let input_path = Path::new(&input);
    if input_path.is_file() {
    } else {
        error!("Can only export from a file")
    }
}

fn metadata(input: String,
            output: Option<String>,
            flag_scheme: Scheme,
            flag_image_format: Option<ImageFormat>,
            flag_grid_callback: Option<String>) {
    let input_path = Path::new(&input);
    if input_path.is_file() {
    } else {
        error!("Can only dumps from a file")
    }
}
<|endoftext|>


<commit_before>#![feature(plugin)]

extern crate "rustc-serialize" as rustc_serialize;
extern crate docopt;
#[plugin] #[no_link] extern crate docopt_macros;

use std::collections::HashSet;

docopt!(Config derive Debug, "
Usage: env-control <string> [-a PATH | -p PATH | -r PATH]...

Options:
    -a, --append PATH   Append this path to the variable
    -p, --prepend PATH  Prepend this path to the variable
    -r, --remove PATH   Remove this path from the variable
    <string>            The PATH-like string to manipulate
", flag_remove: HashSet<String>);

fn main() {
    let cfg: Config = Config::docopt().decode().unwrap_or_else(|e| e.exit());
    let to_append    = cfg.flag_append;
    let to_prepend   = cfg.flag_prepend;
    let to_remove    = cfg.flag_remove;
    let current_path = cfg.arg_string;

    let mut combined_paths = to_prepend.into_iter()
        .chain(current_path.split(':').map(|slice| slice.to_string()))
        .chain(to_append.into_iter());

    let mut printed_paths: HashSet<String> = HashSet::new();

    if let Some(first_path) = combined_paths.next() {
        print!("{}", first_path);
        printed_paths.insert(first_path);
    }

    for path in combined_paths {
        if !printed_paths.contains(&path) && !to_remove.contains(&path) {
            print!(":{}", path);
            printed_paths.insert(path);
        }
    }

    print!("\n");
}
<commit_msg>Ignore empty paths<commit_after>#![feature(plugin)]

extern crate "rustc-serialize" as rustc_serialize;
extern crate docopt;
#[plugin] #[no_link] extern crate docopt_macros;

use std::collections::HashSet;

docopt!(Config derive Debug, "
Usage: env-control <string> [-a PATH | -p PATH | -r PATH]...

Options:
    -a, --append PATH   Append this path to the variable
    -p, --prepend PATH  Prepend this path to the variable
    -r, --remove PATH   Remove this path from the variable
    <string>            The PATH-like string to manipulate
", flag_remove: HashSet<String>);

fn main() {
    let cfg: Config = Config::docopt().decode().unwrap_or_else(|e| e.exit());
    let to_append    = cfg.flag_append;
    let to_prepend   = cfg.flag_prepend;
    let to_remove    = cfg.flag_remove;
    let current_path = cfg.arg_string;

    let mut combined_paths = to_prepend.into_iter()
        .chain(current_path.split(':').map(|slice| slice.to_string()))
        .chain(to_append.into_iter());

    let mut printed_paths: HashSet<String> = HashSet::new();

    match combined_paths.next() {
        Some(first_path) => {
            if !first_path.trim().is_empty() {
                print!("{}", first_path);
                printed_paths.insert(first_path);
            }
        },

        None => return,
    }

    for path in combined_paths {
        if path.trim().is_empty() || printed_paths.contains(&path) || to_remove.contains(&path) {
            continue
        }

        print!(":{}", path);
        printed_paths.insert(path);
    }

    print!("\n");
}
<|endoftext|>


<commit_before>extern crate discord;

use std::env;
use discord::{Discord, State};
use discord::model::Event;

pub fn main() {
	// Log in to Discord using a bot token from the environment
	let discord = Discord::from_bot_token(
		&env::var("DISCORD_TOKEN").expect("Expected token"),
	).expect("login failed");

	// establish websocket and voice connection
	let (mut connection, ready) = discord.connect().expect("connect failed");
	println!("[Ready] {} is serving {} servers", ready.user.username, ready.servers.len());
	let mut state = State::new(ready);
	connection.sync_calls(&state.all_private_channels());

	// receive events forever
	loop {
		let event = match connection.recv_event() {
			Ok(event) => event,
			Err(err) => {
				println!("[Warning] Receive error: {:?}", err);
				if let discord::Error::WebSocket(..) = err {
					// Handle the websocket connection being dropped
					let (new_connection, ready) = discord.connect().expect("connect failed");
					connection = new_connection;
					state = State::new(ready);
					println!("[Ready] Reconnected successfully.");
				}
				if let discord::Error::Closed(..) = err {
					break
				}
				continue
			},
		};
		state.update(&event);

		match event {
			Event::MessageCreate(message) => {
				use std::ascii::AsciiExt;
				// safeguard: stop if the message is from us
				if message.author.id == state.user().id {
					continue
				}

				// reply to a command if there was one
				let mut split = message.content.split(' ');
				let first_word = split.next().unwrap_or("");
				let argument = split.next().unwrap_or("");

				//Section of avaliable commands.
                if first_word.eq_ignore_ascii_case("!help") {
                    if argument.eq_ignore_ascii_case("commands") {
                        warn(discord.send_message(message.channel_id, "help, about, uber, distrowiki, and sourcecode are the avaliable commands. (!dj is buggy)", "", false))
                    }
                    else {
                    warn(discord.send_message(message.channel_id, "Umm, you gotta use it like this. !help [args]", "", false))
                    }
                }

                if first_word.eq_ignore_ascii_case("!about") {
                    warn(discord.send_message(message.channel_id, "I have no clue. All I know is that I was written by Arroz in a language called Rust, and my code is still being developed.", "", false))
                }

                if first_word.eq_ignore_ascii_case("!uber"){
					if argument.eq_ignore_ascii_case("ddog75") {
                        warn(discord.send_message(message.channel_id, "Alright, 10 minutes.", "", false))
                    } else {
                        warn(discord.send_message(message.channel_id, "Fuck off!", "", false))
                    }
                }

                if first_word.eq_ignore_ascii_case("!sourcecode") {
                    warn(discord.send_message(message.channel_id, "Okay, here you go. https://github.com/aarroz/rice_bot2", "", false))
                }
				//Section that prints wikis for linux distributions.
				if first_word.eq_ignore_ascii_case("!distrowiki") {
					if argument.eq_ignore_ascii_case("arch") {
						warn(discord.send_message(message.channel_id, "Here's the Arch Wiki! https://wiki.archlinux.org/", "", false))
					}
					if argument.eq_ignore_ascii_case("ubuntu") {
						warn(discord.send_message(message.channel_id, "Here's the Ubuntu Wiki! https://wiki.ubuntu.com/", "", false))
					}
					if argument.eq_ignore_ascii_case("debian") {
						warn(discord.send_message(message.channel_id, "Here's the Debian Wiki! https://wiki.debian.org/", "", false))
					}
					if argument.eq_ignore_ascii_case("antergos") {
						warn(discord.send_message(message.channel_id, "Here's the Antergos Wiki! https://antergos.com/wiki/", "", false))
					}
					if argument.eq_ignore_ascii_case("centos") {
						warn(discord.send_message(message.channel_id, "Here's the CentOS Wiki! https://wiki.centos.org/", "", false))
					}
					//Sorry OpenBSD users.
					if argument.eq_ignore_ascii_case("openbsd") {
						warn(discord.send_message(message.channel_id, "No, just no.", "", false))
					}
					//Not sorry.
					if argument.eq_ignore_ascii_case("gentoo") {
						warn(discord.send_message(message.channel_id, "Here's the Gentoo Wiki! https://wiki.gentoo.org/wiki/Main_Page", "", false))
					}
					if argument.eq_ignore_ascii_case("opensuse") {
						warn(discord.send_message(message.channel_id, "Here's the OpenSuse Wiki! https://en.opensuse.org/Portal:Wiki", "", false))
					}
				}

                // Command for using dj.
				if first_word.eq_ignore_ascii_case("!dj") {
					let vchan = state.find_voice_user(message.author.id);
					if argument.eq_ignore_ascii_case("stop") {
						vchan.map(|(sid, _)| connection.voice(sid).stop());
					} else if argument.eq_ignore_ascii_case("quit") {
						vchan.map(|(sid, _)| connection.drop_voice(sid));
					} else {
						let output = if let Some((server_id, channel_id)) = vchan {
							match discord::voice::open_ytdl_stream(argument) {
								Ok(stream) => {
									let voice = connection.voice(server_id);
									voice.set_deaf(true);
									voice.connect(channel_id);
									voice.play(stream);
									String::new()
								},
								Err(error) => format!("Error: {}", error),
							}
						} else {
							"You must be in a voice channel to DJ".to_owned()
						};
						if !output.is_empty() {
							warn(discord.send_message(message.channel_id, &output, "", false));
						}
					}
				}
			}
			Event::VoiceStateUpdate(server_id, _) => {
				// If someone moves/hangs up, and we are in a voice channel,
				if let Some(cur_channel) = connection.voice(server_id).current_channel() {
					// and our current voice channel is empty, disconnect from voice
					match server_id {
						Some(server_id) => if let Some(srv) = state.servers().iter().find(|srv| srv.id == server_id) {
							if srv.voice_states.iter().filter(|vs| vs.channel_id == Some(cur_channel)).count() <= 1 {
								connection.voice(Some(server_id)).disconnect();
							}
						},
						None => if let Some(call) = state.calls().get(&cur_channel) {
							if call.voice_states.len() <= 1 {
								connection.voice(server_id).disconnect();
							}
						}
					}
				}
			}
			_ => {}, // discard other events
		}
	}
}

fn warn<T, E: ::std::fmt::Debug>(result: Result<T, E>) {
	match result {
		Ok(_) => {},
		Err(err) => println!("[Warning] {:?}", err)
	}
}
<commit_msg>Added Help arg<commit_after>extern crate discord;

use std::env;
use discord::{Discord, State};
use discord::model::Event;

pub fn main() {
	// Log in to Discord using a bot token from the environment
	let discord = Discord::from_bot_token(
		&env::var("DISCORD_TOKEN").expect("Expected token"),
	).expect("login failed");

	// establish websocket and voice connection
	let (mut connection, ready) = discord.connect().expect("connect failed");
	println!("[Ready] {} is serving {} servers", ready.user.username, ready.servers.len());
	let mut state = State::new(ready);
	connection.sync_calls(&state.all_private_channels());

	// receive events forever
	loop {
		let event = match connection.recv_event() {
			Ok(event) => event,
			Err(err) => {
				println!("[Warning] Receive error: {:?}", err);
				if let discord::Error::WebSocket(..) = err {
					// Handle the websocket connection being dropped
					let (new_connection, ready) = discord.connect().expect("connect failed");
					connection = new_connection;
					state = State::new(ready);
					println!("[Ready] Reconnected successfully.");
				}
				if let discord::Error::Closed(..) = err {
					break
				}
				continue
			},
		};
		state.update(&event);

		match event {
			Event::MessageCreate(message) => {
				use std::ascii::AsciiExt;
				// safeguard: stop if the message is from us
				if message.author.id == state.user().id {
					continue
				}

				// reply to a command if there was one
				let mut split = message.content.split(' ');
				let first_word = split.next().unwrap_or("");
				let argument = split.next().unwrap_or("");

				//Section of avaliable commands.
                if first_word.eq_ignore_ascii_case("!help") {
                    if argument.eq_ignore_ascii_case("commands") {
                        warn(discord.send_message(message.channel_id, "help, about, uber, distrowiki, and sourcecode are the avaliable commands. (!dj is buggy)", "", false))
                    }
		    if argument.eq_ignore_ascii_case("about") {
			 warn(discord.send_message(message.channel_id, "Displays info about me. No arguments available.", "", false))
		    }
		    if argument.eq_ignore_ascii_case("uber") {
			warn(discord.send_message(message.channel_id, "Uber an object (I'm picky).", "", false))
		    }
		    if argument.eq_ignore_ascii_case("distrowiki") {
			warn(discord.send_message(message.channel_id, "Finds the wiki of the specified distro. Arguments include arch, ubuntu, debian, antergos, centos, and openbsd.", "", false))
		    }
		    if argument.eq_ignore_ascii_case("sourcecode") {
			warn(discord.send_message(message.channel_id, "Displays where I am on GitHub. No arguments available", "", false))
	            }
		    if argument.eq_ignore_ascii_case("help") {
			warn(discord.send_message(message.channel_id, "Are you trying to break me?", "", false))
	            }
                    else {
                    warn(discord.send_message(message.channel_id, "Umm, you gotta use it like this. !help [args]", "", false))
                    }
                }

                if first_word.eq_ignore_ascii_case("!about") {
                    warn(discord.send_message(message.channel_id, "I have no clue. All I know is that I was written by Arroz in a language called Rust, and my code is still being developed.", "", false))
                }

                if first_word.eq_ignore_ascii_case("!uber"){
					if argument.eq_ignore_ascii_case("ddog75") {
                        warn(discord.send_message(message.channel_id, "Alright, 10 minutes.", "", false))
                    } else {
                        warn(discord.send_message(message.channel_id, "Fuck off!", "", false))
                    }
                }

                if first_word.eq_ignore_ascii_case("!sourcecode") {
                    warn(discord.send_message(message.channel_id, "Okay, here you go. https://github.com/aarroz/rice_bot2", "", false))
                }
				//Section that prints wikis for linux distributions.
				if first_word.eq_ignore_ascii_case("!distrowiki") {
					if argument.eq_ignore_ascii_case("arch") {
						warn(discord.send_message(message.channel_id, "Here's the Arch Wiki! https://wiki.archlinux.org/", "", false))
					}
					if argument.eq_ignore_ascii_case("ubuntu") {
						warn(discord.send_message(message.channel_id, "Here's the Ubuntu Wiki! https://wiki.ubuntu.com/", "", false))
					}
					if argument.eq_ignore_ascii_case("debian") {
						warn(discord.send_message(message.channel_id, "Here's the Debian Wiki! https://wiki.debian.org/", "", false))
					}
					if argument.eq_ignore_ascii_case("antergos") {
						warn(discord.send_message(message.channel_id, "Here's the Antergos Wiki! https://antergos.com/wiki/", "", false))
					}
					if argument.eq_ignore_ascii_case("centos") {
						warn(discord.send_message(message.channel_id, "Here's the CentOS Wiki! https://wiki.centos.org/", "", false))
					}
					//Sorry OpenBSD users.
					if argument.eq_ignore_ascii_case("openbsd") {
						warn(discord.send_message(message.channel_id, "No, just no.", "", false))
					}
					//Not sorry.
					if argument.eq_ignore_ascii_case("gentoo") {
						warn(discord.send_message(message.channel_id, "Here's the Gentoo Wiki! https://wiki.gentoo.org/wiki/Main_Page", "", false))
					}
					if argument.eq_ignore_ascii_case("opensuse") {
						warn(discord.send_message(message.channel_id, "Here's the OpenSuse Wiki! https://en.opensuse.org/Portal:Wiki", "", false))
					}
				}

                // Command for using dj.
				if first_word.eq_ignore_ascii_case("!dj") {
					let vchan = state.find_voice_user(message.author.id);
					if argument.eq_ignore_ascii_case("stop") {
						vchan.map(|(sid, _)| connection.voice(sid).stop());
					} else if argument.eq_ignore_ascii_case("quit") {
						vchan.map(|(sid, _)| connection.drop_voice(sid));
					} else {
						let output = if let Some((server_id, channel_id)) = vchan {
							match discord::voice::open_ytdl_stream(argument) {
								Ok(stream) => {
									let voice = connection.voice(server_id);
									voice.set_deaf(true);
									voice.connect(channel_id);
									voice.play(stream);
									String::new()
								},
								Err(error) => format!("Error: {}", error),
							}
						} else {
							"You must be in a voice channel to DJ".to_owned()
						};
						if !output.is_empty() {
							warn(discord.send_message(message.channel_id, &output, "", false));
						}
					}
				}
			}
			Event::VoiceStateUpdate(server_id, _) => {
				// If someone moves/hangs up, and we are in a voice channel,
				if let Some(cur_channel) = connection.voice(server_id).current_channel() {
					// and our current voice channel is empty, disconnect from voice
					match server_id {
						Some(server_id) => if let Some(srv) = state.servers().iter().find(|srv| srv.id == server_id) {
							if srv.voice_states.iter().filter(|vs| vs.channel_id == Some(cur_channel)).count() <= 1 {
								connection.voice(Some(server_id)).disconnect();
							}
						},
						None => if let Some(call) = state.calls().get(&cur_channel) {
							if call.voice_states.len() <= 1 {
								connection.voice(server_id).disconnect();
							}
						}
					}
				}
			}
			_ => {}, // discard other events
		}
	}
}

fn warn<T, E: ::std::fmt::Debug>(result: Result<T, E>) {
	match result {
		Ok(_) => {},
		Err(err) => println!("[Warning] {:?}", err)
	}
}
<|endoftext|>


<commit_before>
extern crate chrono;
extern crate docopt;

use std::process::Command;

use std::io::prelude::*;
use std::net::TcpStream;

use docopt::Docopt;

extern crate libc;
use libc::*;
use std::ffi::CString;
use std::default::Default;

// #define MFSTYPENAMELEN  16  length of fs type name including null
// #define MAXPATHLEN      1024
// #define MNAMELEN        MAXPATHLEN
   // struct statfs { /* when _DARWIN_FEATURE_64_BIT_INODE is defined */
   //       uint32_t    f_bsize;        /* fundamental file system block size */
   //       int32_t     f_iosize;       /* optimal transfer block size */
   //       uint64_t    f_blocks;       /* total data blocks in file system */
   //       uint64_t    f_bfree;        /* free blocks in fs */
   //       uint64_t    f_bavail;       /* free blocks avail to non-superuser */
   //       uint64_t    f_files;        /* total file nodes in file system */
   //       uint64_t    f_ffree;        /* free file nodes in fs */
   //       fsid_t      f_fsid;         /* file system id */
   //       uid_t       f_owner;        /* user that mounted the filesystem */
   //       uint32_t    f_type;         /* type of filesystem */
   //       uint32_t    f_flags;        /* copy of mount exported flags */
   //       uint32_t    f_fssubtype;    /* fs sub-type (flavor) */
   //       char        f_fstypename[MFSTYPENAMELEN];   /* fs type name */
   //       char        f_mntonname[MAXPATHLEN];        /* directory on which mounted */
   //       char        f_mntfromname[MAXPATHLEN];      /* mounted filesystem */
   //       uint32_t    f_reserved[8];  /* For future use */
   //   };

type Uid = uint32_t;

#[repr(C)]
struct StatFs {
	f_bsize: uint32_t,
	f_iosize: int32_t,
	f_blocks: uint64_t,
	f_bfree:  uint64_t,
	f_bavail: uint64_t,
	f_files:  uint64_t,
	f_ffree:  uint64_t,
	f_fsid:	  [uint32_t; 2],
	f_owner:  Uid,
	f_type:	  uint32_t,
	f_flags:  uint32_t,
	f_fstypename: 	[c_char; 16],
	f_mntonname: 	[c_char; 1024],
	f_mntfromname: 	[c_char; 1024],
	f_reserved: 	[uint32_t; 8],
}

impl Default for StatFs {
	fn default() -> StatFs {
		StatFs{
			f_bsize: 0,
			f_iosize: 0,
			f_blocks: 0,
			f_bfree: 0,
			f_bavail: 0,
			f_files: 0,
			f_ffree: 0,
			f_fsid: [0; 2],
			f_owner: 0,
			f_type: 0,
			f_flags: 0,
			f_fstypename: [0; 16],
			f_mntonname:  [0; 1024],
			f_mntfromname:[0; 1024],
			f_reserved: [0; 8]
		}
	}
}

extern {
	fn statfs(path: *const c_char, stafs: *mut StatFs) -> size_t;
	// statfs(const char *path, struct statfs *buf);
}

// Write the Docopt usage string. dfrites ?
static USAGE: &'static str = "
Usage: durite -g GHOSTNAME -l HOSTNAME [-p PORT]
       durite (--help | -h)

Options:
    -h, --help     Show this screen.
    -l HOSTNAME    Hostname to advertise in graphite.
    -g GHOSTNAME   Graphite Hostname.
    -p PORT        Graphite port [default: 2003]
";

#[cfg(not(any(target_os="macos")))]
fn disk_free() -> std::process::Output {
	Command::new("df")
		.arg("--portability").output().unwrap_or_else(|e| { panic!("failed to execute process: {}", e) })
}

#[cfg(any(target_os="macos"))]
fn disk_free() -> std::process::Output {
	Command::new("df")
		.arg("-k").output().unwrap_or_else(|e| { panic!("failed to execute process: {}", e) })
}

fn send_content(stream: &mut std::net::TcpStream, hostname: &str, probe_and_value: String, timestamp: i64) {
	let content = format!("durite.{}.{} {}\n", hostname, probe_and_value, timestamp);
	let _ = stream.write(&content.as_bytes());
}

fn main() {

	// Parse argv and exit the program with an error message if it fails.
	let args = Docopt::new(USAGE)
		.and_then(|d| d.argv(std::env::args().into_iter()).parse())
		.unwrap_or_else(|e| e.exit());

    let connect_string = format!("{}:{}", args.get_str("-g"), args.get_str("-p"));
    println!("Connect string is {}", connect_string);
    let my_hostname = args.get_str("-l");

	while true {
	    let dt = chrono::UTC::now();
	    let timestamp = dt.timestamp();
	    let o = disk_free();
		let stdout = String::from_utf8(o.stdout).
			ok().
			unwrap();

		let lines = stdout.split("\n");

	    match TcpStream::connect(&*connect_string) {
	    	Ok(s) => {
	    		let mut stream = s;
				for line in lines {
					if line.starts_with("/") {
						let values: Vec<&str> = line.split(" ").filter(|e| {
							e.len() != 0
						}).collect();

						let disk = values[0];
						let all = values[2];
						let available = values[3];
						// let w = values[8].to_string();
						// let mut st: StatFs = Default::default();
						// let mp = CString::new(w.into_bytes()).unwrap();
						// unsafe {
						// 	let o = statfs(mp.as_ptr(), &mut st);
						// 	println!("o{} {:?}", o, st.f_bfree);
						// }
						send_content(&mut stream, my_hostname, format!("{}.available {}", disk, available), timestamp);
						send_content(&mut stream, my_hostname, format!("{}.all {}", disk, all), timestamp);
				 	}
				 }
	    	},
	    	Err(e) => println!("Unable to connect to {}: {:?}\nIgnoring this data point.", connect_string, e)
		}
	 	std::thread::sleep_ms(10000);
	}
}<commit_msg>Report correct unit<commit_after>
extern crate chrono;
extern crate docopt;

use std::process::Command;

use std::io::prelude::*;
use std::net::TcpStream;

use docopt::Docopt;

extern crate libc;
use libc::*;
use std::ffi::CString;
use std::default::Default;

// #define MFSTYPENAMELEN  16  length of fs type name including null
// #define MAXPATHLEN      1024
// #define MNAMELEN        MAXPATHLEN
   // struct statfs { /* when _DARWIN_FEATURE_64_BIT_INODE is defined */
   //       uint32_t    f_bsize;        /* fundamental file system block size */
   //       int32_t     f_iosize;       /* optimal transfer block size */
   //       uint64_t    f_blocks;       /* total data blocks in file system */
   //       uint64_t    f_bfree;        /* free blocks in fs */
   //       uint64_t    f_bavail;       /* free blocks avail to non-superuser */
   //       uint64_t    f_files;        /* total file nodes in file system */
   //       uint64_t    f_ffree;        /* free file nodes in fs */
   //       fsid_t      f_fsid;         /* file system id */
   //       uid_t       f_owner;        /* user that mounted the filesystem */
   //       uint32_t    f_type;         /* type of filesystem */
   //       uint32_t    f_flags;        /* copy of mount exported flags */
   //       uint32_t    f_fssubtype;    /* fs sub-type (flavor) */
   //       char        f_fstypename[MFSTYPENAMELEN];   /* fs type name */
   //       char        f_mntonname[MAXPATHLEN];        /* directory on which mounted */
   //       char        f_mntfromname[MAXPATHLEN];      /* mounted filesystem */
   //       uint32_t    f_reserved[8];  /* For future use */
   //   };

type Uid = uint32_t;

#[repr(C)]
struct StatFs {
	f_bsize: uint32_t,
	f_iosize: int32_t,
	f_blocks: uint64_t,
	f_bfree:  uint64_t,
	f_bavail: uint64_t,
	f_files:  uint64_t,
	f_ffree:  uint64_t,
	f_fsid:	  [uint32_t; 2],
	f_owner:  Uid,
	f_type:	  uint32_t,
	f_flags:  uint32_t,
	f_fstypename: 	[c_char; 16],
	f_mntonname: 	[c_char; 1024],
	f_mntfromname: 	[c_char; 1024],
	f_reserved: 	[uint32_t; 8],
}

impl Default for StatFs {
	fn default() -> StatFs {
		StatFs{
			f_bsize: 0,
			f_iosize: 0,
			f_blocks: 0,
			f_bfree: 0,
			f_bavail: 0,
			f_files: 0,
			f_ffree: 0,
			f_fsid: [0; 2],
			f_owner: 0,
			f_type: 0,
			f_flags: 0,
			f_fstypename: [0; 16],
			f_mntonname:  [0; 1024],
			f_mntfromname:[0; 1024],
			f_reserved: [0; 8]
		}
	}
}

extern {
	fn statfs(path: *const c_char, stafs: *mut StatFs) -> size_t;
	// statfs(const char *path, struct statfs *buf);
}

// Write the Docopt usage string. dfrites ?
static USAGE: &'static str = "
Usage: durite -g GHOSTNAME -l HOSTNAME [-p PORT]
       durite (--help | -h)

Options:
    -h, --help     Show this screen.
    -l HOSTNAME    Hostname to advertise in graphite.
    -g GHOSTNAME   Graphite Hostname.
    -p PORT        Graphite port [default: 2003]
";

static VERSION: &'static str = "0.0.1"

#[cfg(not(any(target_os="macos")))]
fn disk_free() -> std::process::Output {
	Command::new("df")
		.arg("--portability").output().unwrap_or_else(|e| { panic!("failed to execute process: {}", e) })
}

#[cfg(any(target_os="macos"))]
fn disk_free() -> std::process::Output {
	Command::new("df")
		.arg("-k").output().unwrap_or_else(|e| { panic!("failed to execute process: {}", e) })
}

fn send_content(stream: &mut std::net::TcpStream, hostname: &str, probe_and_value: String, timestamp: i64) {
	let content = format!("durite.{}.{} {}\n", hostname, probe_and_value, timestamp);
	let _ = stream.write(&content.as_bytes());
}

fn main() {

	// Parse argv and exit the program with an error message if it fails.
	let args = Docopt::new(USAGE)
		.and_then(|d| d.argv(std::env::args().into_iter()).parse())
		.unwrap_or_else(|e| e.exit());

    let connect_string = format!("{}:{}", args.get_str("-g"), args.get_str("-p"));
    println!("Connect string is {}", connect_string);
    let my_hostname = args.get_str("-l");

	while true {
	    let dt = chrono::UTC::now();
	    let timestamp = dt.timestamp();
	    let o = disk_free();
		let stdout = String::from_utf8(o.stdout).
			ok().
			unwrap();

		let lines = stdout.split("\n");

	    match TcpStream::connect(&*connect_string) {
	    	Ok(s) => {
	    		let mut stream = s;
				for line in lines {
					if line.starts_with("/") {
						let values: Vec<&str> = line.split(" ").filter(|e| {
							e.len() != 0
						}).collect();

						let disk = values[0];
						// du reports kbytes
						let all = values[2] * 1024;
						let available = values[3] *1024;
						// let w = values[8].to_string();
						// let mut st: StatFs = Default::default();
						// let mp = CString::new(w.into_bytes()).unwrap();
						// unsafe {
						// 	let o = statfs(mp.as_ptr(), &mut st);
						// 	println!("o{} {:?}", o, st.f_bfree);
						// }
						send_content(&mut stream, my_hostname, format!("{}.available {}", disk, available), timestamp);
						send_content(&mut stream, my_hostname, format!("{}.all {}", disk, all), timestamp);
				 	}
				 }
	    	},
	    	Err(e) => println!("Unable to connect to {}: {:?}\nIgnoring this data point.", connect_string, e)
		}
	 	std::thread::sleep_ms(10000);
	}
}<|endoftext|>


<commit_before>fn main() {
    println!("Hello, world!");
}
<commit_msg>Basic outline of event loop<commit_after>use crossbeam_channel::{bounded, Sender};
use std::io;
use std::thread;

fn input_thread(s: Sender<u32>) {
    loop {
        let mut buffer = String::new();
        io::stdin().read_line(&mut buffer).unwrap();
        let i = buffer.trim().parse::<u32>().unwrap();
        s.send(i).unwrap();
    }
}

fn main() {
    let (sender, receiver) = bounded(32);

    let s = sender.clone();
    thread::Builder::new()
        .name("input".to_string())
        .spawn(move || {
            input_thread(s);
        })
        .unwrap();

    drop(sender);

    loop {
        match receiver.recv().unwrap() {
            0 => break,
            x => println!("received {}", x),
        }
    }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate peresil;

use std::fs::File;
use std::io::prelude::*;

use peresil::{ParseMaster, StringPoint, Progress, Status, Recoverable};

#[derive(Debug, Copy, Clone)]
enum Error {
    ExpectedHeader,
    ExpectedInbox,
    ExpectedOutbox,
    ExpectedCopyFrom,
    ExpectedCopyTo,
    ExpectedBumpUp,
    ExpectedBumpDown,
    ExpectedAdd,
    ExpectedSub,
    ExpectedIndirectRegister,
    ExpectedIndirectRegisterEnd,
    ExpectedRegisterValue,
    ExpectedLabelDefinition,
    ExpectedLabelValue,
    ExpectedJump,
    ExpectedJumpIfZero,
    ExpectedJumpIfNegative,
    ExpectedWhiteSpace,
    ExpectedComment,
    ExpectedCommentId,
    ExpectedCommentDefinition,
    ExpectedCommentDefinitionData,
    ExpectedCommentDefinitionEnd,
    ExpectedColon,
}

impl Recoverable for Error {
    fn recoverable(&self) -> bool { true }
}

type ZPM<'a> = ParseMaster<StringPoint<'a>, Error>;
type ZPR<'a, T> = Progress<StringPoint<'a>, T, Error>;

struct Thing<'a> {
    point: StringPoint<'a>,
}

impl<'a> Thing<'a> {
    fn new(s: &str) -> Thing {
        Thing {
            point: StringPoint::new(s),
        }
    }
}

#[derive(Debug, Copy, Clone)]
enum Register {
    Direct(u8),
    Indirect(u8),
}

type Label<'a> = &'a str;
type CommentId<'a> = &'a str;
type CommentData<'a> = &'a str;

#[derive(Debug, Copy, Clone)]
enum Token<'a> {
    Header,
    Inbox,
    Outbox,
    CopyFrom(Register),
    CopyTo(Register),
    BumpUp(Register),
    BumpDown(Register), // name?
    Add(Register),
    Sub(Register),
    LabelDefinition(Label<'a>),
    Jump(Label<'a>),
    JumpIfZero(Label<'a>),
    JumpIfNegative(Label<'a>),
    // negative?
    Comment(CommentId<'a>),
    CommentDefinition(CommentId<'a>, CommentData<'a>),
    Whitespace(&'a str),
}

fn parse_header<'a>(_: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    pt.consume_literal("-- HUMAN RESOURCE MACHINE PROGRAM --")
        .map(|_| Token::Header)
        .map_err(|_| Error::ExpectedHeader)
}

fn parse_inbox<'a>(_: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    pt.consume_literal("INBOX")
        .map(|_| Token::Inbox)
        .map_err(|_| Error::ExpectedInbox)
}

fn parse_outbox<'a>(_: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    pt.consume_literal("OUTBOX")
        .map(|_| Token::Outbox)
        .map_err(|_| Error::ExpectedOutbox)
}

fn parse_copy_from<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    let (pt, _) = try_parse!{
        pt.consume_literal("COPYFROM")
            .map_err(|_| Error::ExpectedCopyFrom)
    };

    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};

    let (pt, reg) = try_parse!{parse_register(pm, pt)};

    Progress::success(pt, Token::CopyFrom(reg))
}

fn parse_copy_to<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    let (pt, _) = try_parse!{
        pt.consume_literal("COPYTO")
            .map_err(|_| Error::ExpectedCopyTo)
    };

    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};

    let (pt, reg) = try_parse!{parse_register(pm, pt)};

    Progress::success(pt, Token::CopyTo(reg))
}

fn parse_bump_up<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    let (pt, _) = try_parse!{
        pt.consume_literal("BUMPUP")
            .map_err(|_| Error::ExpectedBumpUp)
    };

    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};

    let (pt, reg) = try_parse!{parse_register(pm, pt)};

    Progress::success(pt, Token::BumpUp(reg))
}

// Single register stuff is very repeated
fn parse_bump_down<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    let (pt, _) = try_parse!{
        pt.consume_literal("BUMPDOWN")
            .map_err(|_| Error::ExpectedBumpDown)
    };

    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};

    let (pt, reg) = try_parse!{parse_register(pm, pt)};

    Progress::success(pt, Token::BumpDown(reg))
}

fn parse_add<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    let (pt, _) = try_parse!{
        pt.consume_literal("ADD")
            .map_err(|_| Error::ExpectedAdd)
    };

    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};

    let (pt, reg) = try_parse!{parse_register(pm, pt)};

    Progress::success(pt, Token::Add(reg))
}

fn parse_sub<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    let (pt, _) = try_parse!{
        pt.consume_literal("SUB")
            .map_err(|_| Error::ExpectedSub)
    };

    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};

    let (pt, reg) = try_parse!{parse_register(pm, pt)};

    Progress::success(pt, Token::Sub(reg))
}

fn parse_register<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Register> {
    pm.alternate()
        .one(|pm| parse_register_indirect(pm, pt))
        .one(|pm| parse_register_value(pm, pt).map(Register::Direct))
        .finish()
}

fn parse_register_indirect<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Register> {
    let (pt, _) = try_parse!{
        pt.consume_literal("[").map_err(|_| Error::ExpectedIndirectRegister)
    };
    let (pt, reg) = try_parse!(parse_register_value(pm, pt));
    let (pt, _) = try_parse!{
        pt.consume_literal("]").map_err(|_| Error::ExpectedIndirectRegisterEnd)
    };

    Progress::success(pt, Register::Indirect(reg))
}

fn parse_register_value<'a>(_: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, u8> {
    let end = match pt.s.char_indices().skip_while(|&(_, c)| c.is_digit(10)).next() {
        Some((pos, _)) if pos == 0 => None,
        Some((pos, _)) => Some(pos),
        None => Some(pt.s.len()),
    };
    pt.consume_to(end)
        .map(|v| v.parse().unwrap()) // unrecoverable error
        .map_err(|_| Error::ExpectedRegisterValue)
}

fn parse_label_definition<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    let (pt, val) = try_parse!{
        parse_label_value(pm, pt)
            .map_err(|_| Error::ExpectedLabelDefinition)
    };
    let (pt, _) = try_parse!(pt.consume_literal(":").map_err(|_| Error::ExpectedColon));

    Progress::success(pt, Token::LabelDefinition(val))
}

fn parse_label_value<'a>(_: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, &'a str> {
    let end = match pt.s.char_indices().skip_while(|&(_, c)| c >= 'a' && c <= 'z').next() {
        Some((pos, _)) if pos == 0 => None,
        Some((pos, _)) => Some(pos),
        None => Some(pt.s.len()),
    };

    pt.consume_to(end)
        .map_err(|_| Error::ExpectedLabelValue)
}

fn parse_jump<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    let (pt, _) = try_parse!(
        pt.consume_literal("JUMP")
            .map_err(|_| Error::ExpectedJump)
    );

    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};

    let (pt, lab) = try_parse!{parse_label_value(pm, pt)};

    Progress::success(pt, Token::Jump(lab))
}

fn parse_jump_if_zero<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    let (pt, _) = try_parse!(
        pt.consume_literal("JUMPZ")
            .map_err(|_| Error::ExpectedJumpIfZero)
    );

    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};

    let (pt, lab) = try_parse!{parse_label_value(pm, pt)};

    Progress::success(pt, Token::JumpIfZero(lab))
}

fn parse_jump_if_negative<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    let (pt, _) = try_parse!(
        pt.consume_literal("JUMPN")
            .map_err(|_| Error::ExpectedJumpIfNegative)
    );

    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};

    let (pt, lab) = try_parse!{parse_label_value(pm, pt)};

    Progress::success(pt, Token::JumpIfNegative(lab))
}

fn parse_comment<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    let (pt, _) = try_parse!(
        pt.consume_literal("COMMENT")
            .map_err(|_| Error::ExpectedComment)
    );

    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};

    let (pt, id) = try_parse!{parse_comment_id(pm, pt)};

    Progress::success(pt, Token::Comment(id))
}

fn parse_comment_definition<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    let (pt, _) = try_parse!(
        pt.consume_literal("DEFINE COMMENT")
            .map_err(|_| Error::ExpectedCommentDefinition)
    );

    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};

    let (pt, id) = try_parse!{parse_comment_id(pm, pt)};

    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};

    let (pt, data) = try_parse!{parse_comment_data(pm, pt)};

    let (pt, _) = try_parse!{
        pt.consume_literal(";")
            .map_err(|_| Error::ExpectedCommentDefinitionEnd)
    };

    Progress::success(pt, Token::CommentDefinition(id, data))
}

fn parse_comment_id<'a>(_: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, &'a str> {
// Duplicated logic - check and pull to peresil?
    let end = match pt.s.char_indices().skip_while(|&(_, c)| c.is_digit(10)).next() {
        Some((pos, _)) if pos == 0 => None,
        Some((pos, _)) => Some(pos),
        None => Some(pt.s.len()),
    };
    pt.consume_to(end)
        .map_err(|_| Error::ExpectedCommentId)
}

fn parse_comment_data<'a>(_: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, &'a str> {
// Duplicated logic - check and pull to peresil?
    let end = match pt.s.char_indices().skip_while(|&(_, c)| c != ';').next() {
        Some((pos, _)) if pos == 0 => None,
        Some((pos, _)) => Some(pos),
        None => Some(pt.s.len()),
    };
    pt.consume_to(end)
        .map_err(|_| Error::ExpectedCommentDefinitionData)
}

fn parse_whitespace<'a>(_: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    // Duplicated logic - check and pull to peresil?
    let end = match pt.s.char_indices().skip_while(|&(_, c)| c.is_whitespace()).next() {
        Some((pos, _)) if pos == 0 => None,
        Some((pos, _)) => Some(pos),
        None => Some(pt.s.len()),
    };
    pt.consume_to(end)
        .map(Token::Whitespace)
        .map_err(|_| Error::ExpectedWhiteSpace)
}

impl<'a> Iterator for Thing<'a> {
    type Item = Token<'a>;

    fn next(&mut self) -> Option<Self::Item> {
        let pt = self.point;

        if pt.s.is_empty() { return None }

        let mut pm = ParseMaster::new();

        let tmp = pm.alternate()
            .one(|pm| parse_header(pm, pt))
            .one(|pm| parse_inbox(pm, pt))
            .one(|pm| parse_outbox(pm, pt))
            .one(|pm| parse_copy_from(pm, pt))
            .one(|pm| parse_copy_to(pm, pt))
            .one(|pm| parse_bump_up(pm, pt))
            .one(|pm| parse_bump_down(pm, pt))
            .one(|pm| parse_add(pm, pt))
            .one(|pm| parse_sub(pm, pt))
            .one(|pm| parse_label_definition(pm, pt))
            .one(|pm| parse_jump(pm, pt)) // check ordering of jump (prefix-based?)
            .one(|pm| parse_jump_if_zero(pm, pt))
            .one(|pm| parse_jump_if_negative(pm, pt))
            .one(|pm| parse_comment(pm, pt))
            .one(|pm| parse_comment_definition(pm, pt))
            .one(|pm| parse_whitespace(pm, pt))
            .finish();

        match pm.finish(tmp) {
            Progress { status: Status::Success(tok), point } => {
                self.point = point;
                Some(tok)
            },
            Progress { status: Status::Failure(x), point } => {
                println!("Actually an error: {:?}, {:?}", x, point);
                None
            }
        }
    }
}

fn main() {
    let mut f = File::open("36-alphabetizer-both.txt").expect("File?");

    let mut s = String::new();
    f.read_to_string(&mut s).expect("read");

    let t = Thing::new(&s);

    for t in t {
        if let Token::Whitespace(..) = t { continue }
        println!("{:?}", t);
    }
}
<commit_msg>Reduce duplication of instruction parsers<commit_after>#[macro_use]
extern crate peresil;

use std::fs::File;
use std::io::prelude::*;

use peresil::{ParseMaster, StringPoint, Progress, Status, Recoverable};

#[derive(Debug, Copy, Clone)]
enum Error {
    ExpectedHeader,
    ExpectedInbox,
    ExpectedOutbox,
    ExpectedCopyFrom,
    ExpectedCopyTo,
    ExpectedBumpUp,
    ExpectedBumpDown,
    ExpectedAdd,
    ExpectedSub,
    ExpectedIndirectRegister,
    ExpectedIndirectRegisterEnd,
    ExpectedRegisterValue,
    ExpectedLabelDefinition,
    ExpectedLabelValue,
    ExpectedJump,
    ExpectedJumpIfZero,
    ExpectedJumpIfNegative,
    ExpectedWhiteSpace,
    ExpectedComment,
    ExpectedCommentId,
    ExpectedCommentDefinition,
    ExpectedCommentDefinitionData,
    ExpectedCommentDefinitionEnd,
    ExpectedColon,
}

impl Recoverable for Error {
    fn recoverable(&self) -> bool { true }
}

type ZPM<'a> = ParseMaster<StringPoint<'a>, Error>;
type ZPR<'a, T> = Progress<StringPoint<'a>, T, Error>;

struct Thing<'a> {
    point: StringPoint<'a>,
}

impl<'a> Thing<'a> {
    fn new(s: &str) -> Thing {
        Thing {
            point: StringPoint::new(s),
        }
    }
}

#[derive(Debug, Copy, Clone)]
enum Register {
    Direct(u8),
    Indirect(u8),
}

type Label<'a> = &'a str;
type CommentId<'a> = &'a str;
type CommentData<'a> = &'a str;

#[derive(Debug, Copy, Clone)]
enum Token<'a> {
    Header,
    Inbox,
    Outbox,
    CopyFrom(Register),
    CopyTo(Register),
    BumpUp(Register),
    BumpDown(Register), // name?
    Add(Register),
    Sub(Register),
    LabelDefinition(Label<'a>),
    Jump(Label<'a>),
    JumpIfZero(Label<'a>),
    JumpIfNegative(Label<'a>),
    // negative?
    Comment(CommentId<'a>),
    CommentDefinition(CommentId<'a>, CommentData<'a>),
    Whitespace(&'a str),
}

fn parse_header<'a>(_: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    pt.consume_literal("-- HUMAN RESOURCE MACHINE PROGRAM --")
        .map(|_| Token::Header)
        .map_err(|_| Error::ExpectedHeader)
}

fn parse_inbox<'a>(_: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    pt.consume_literal("INBOX")
        .map(|_| Token::Inbox)
        .map_err(|_| Error::ExpectedInbox)
}

fn parse_outbox<'a>(_: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    pt.consume_literal("OUTBOX")
        .map(|_| Token::Outbox)
        .map_err(|_| Error::ExpectedOutbox)
}

fn parse_copy_from<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    parse_single_register_instruction(pm, pt, "COPYFROM", Token::CopyFrom, Error::ExpectedCopyFrom)
}

fn parse_copy_to<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    parse_single_register_instruction(pm, pt, "COPYTO", Token::CopyTo, Error::ExpectedCopyTo)
}

fn parse_bump_up<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    parse_single_register_instruction(pm, pt, "BUMPUP", Token::BumpUp, Error::ExpectedBumpUp)
}

fn parse_bump_down<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    parse_single_register_instruction(pm, pt, "BUMPDOWN", Token::BumpDown, Error::ExpectedBumpDown)
}

fn parse_add<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    parse_single_register_instruction(pm, pt, "ADD", Token::Add, Error::ExpectedAdd)
}

fn parse_sub<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    parse_single_register_instruction(pm, pt, "SUB", Token::Sub, Error::ExpectedSub)
}

fn parse_single_register_instruction<'a, F>(
    pm: &mut ZPM<'a>,
    pt: StringPoint<'a>,
    instruction_name: &str,
    token_creator: F,
    error_kind: Error
)
    -> ZPR<'a, Token<'a>>
    where F: FnOnce(Register) -> Token<'a>
{
    let (pt, _) = try_parse!(pt.consume_literal(instruction_name).map_err(|_| error_kind));
    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};
    let (pt, reg) = try_parse!{parse_register(pm, pt)};

    Progress::success(pt, token_creator(reg))
}

fn parse_register<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Register> {
    pm.alternate()
        .one(|pm| parse_register_indirect(pm, pt))
        .one(|pm| parse_register_value(pm, pt).map(Register::Direct))
        .finish()
}

fn parse_register_indirect<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Register> {
    let (pt, _) = try_parse!{
        pt.consume_literal("[").map_err(|_| Error::ExpectedIndirectRegister)
    };
    let (pt, reg) = try_parse!(parse_register_value(pm, pt));
    let (pt, _) = try_parse!{
        pt.consume_literal("]").map_err(|_| Error::ExpectedIndirectRegisterEnd)
    };

    Progress::success(pt, Register::Indirect(reg))
}

fn parse_register_value<'a>(_: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, u8> {
    let end = match pt.s.char_indices().skip_while(|&(_, c)| c.is_digit(10)).next() {
        Some((pos, _)) if pos == 0 => None,
        Some((pos, _)) => Some(pos),
        None => Some(pt.s.len()),
    };
    pt.consume_to(end)
        .map(|v| v.parse().unwrap()) // unrecoverable error
        .map_err(|_| Error::ExpectedRegisterValue)
}

fn parse_label_definition<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    let (pt, val) = try_parse!{
        parse_label_value(pm, pt)
            .map_err(|_| Error::ExpectedLabelDefinition)
    };
    let (pt, _) = try_parse!(pt.consume_literal(":").map_err(|_| Error::ExpectedColon));

    Progress::success(pt, Token::LabelDefinition(val))
}

fn parse_label_value<'a>(_: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, &'a str> {
    let end = match pt.s.char_indices().skip_while(|&(_, c)| c >= 'a' && c <= 'z').next() {
        Some((pos, _)) if pos == 0 => None,
        Some((pos, _)) => Some(pos),
        None => Some(pt.s.len()),
    };

    pt.consume_to(end)
        .map_err(|_| Error::ExpectedLabelValue)
}

fn parse_jump<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    parse_jump_instruction(pm, pt, "JUMP", Token::Jump, Error::ExpectedJump)
}

fn parse_jump_if_zero<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    parse_jump_instruction(pm, pt, "JUMPZ", Token::JumpIfZero, Error::ExpectedJumpIfZero)
}

fn parse_jump_if_negative<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    parse_jump_instruction(pm, pt, "JUMPN", Token::JumpIfNegative, Error::ExpectedJumpIfNegative)
}

fn parse_jump_instruction<'a, F>(
    pm: &mut ZPM<'a>,
    pt: StringPoint<'a>,
    instruction_name: &str,
    token_creator: F,
    error_kind: Error
)
    -> ZPR<'a, Token<'a>>
    where F: FnOnce(Label<'a>) -> Token<'a>
{
    let (pt, _) = try_parse!(pt.consume_literal(instruction_name).map_err(|_| error_kind));
    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};
    let (pt, lab) = try_parse!{parse_label_value(pm, pt)};

    Progress::success(pt, token_creator(lab))
}

fn parse_comment<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    let (pt, _) = try_parse!(
        pt.consume_literal("COMMENT")
            .map_err(|_| Error::ExpectedComment)
    );

    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};

    let (pt, id) = try_parse!{parse_comment_id(pm, pt)};

    Progress::success(pt, Token::Comment(id))
}

fn parse_comment_definition<'a>(pm: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    let (pt, _) = try_parse!(
        pt.consume_literal("DEFINE COMMENT")
            .map_err(|_| Error::ExpectedCommentDefinition)
    );

    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};

    let (pt, id) = try_parse!{parse_comment_id(pm, pt)};

    let (pt, _) = try_parse!{parse_whitespace(pm, pt)};

    let (pt, data) = try_parse!{parse_comment_data(pm, pt)};

    let (pt, _) = try_parse!{
        pt.consume_literal(";")
            .map_err(|_| Error::ExpectedCommentDefinitionEnd)
    };

    Progress::success(pt, Token::CommentDefinition(id, data))
}

fn parse_comment_id<'a>(_: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, &'a str> {
// Duplicated logic - check and pull to peresil?
    let end = match pt.s.char_indices().skip_while(|&(_, c)| c.is_digit(10)).next() {
        Some((pos, _)) if pos == 0 => None,
        Some((pos, _)) => Some(pos),
        None => Some(pt.s.len()),
    };
    pt.consume_to(end)
        .map_err(|_| Error::ExpectedCommentId)
}

fn parse_comment_data<'a>(_: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, &'a str> {
// Duplicated logic - check and pull to peresil?
    let end = match pt.s.char_indices().skip_while(|&(_, c)| c != ';').next() {
        Some((pos, _)) if pos == 0 => None,
        Some((pos, _)) => Some(pos),
        None => Some(pt.s.len()),
    };
    pt.consume_to(end)
        .map_err(|_| Error::ExpectedCommentDefinitionData)
}

fn parse_whitespace<'a>(_: &mut ZPM<'a>, pt: StringPoint<'a>) -> ZPR<'a, Token<'a>> {
    // Duplicated logic - check and pull to peresil?
    let end = match pt.s.char_indices().skip_while(|&(_, c)| c.is_whitespace()).next() {
        Some((pos, _)) if pos == 0 => None,
        Some((pos, _)) => Some(pos),
        None => Some(pt.s.len()),
    };
    pt.consume_to(end)
        .map(Token::Whitespace)
        .map_err(|_| Error::ExpectedWhiteSpace)
}

impl<'a> Iterator for Thing<'a> {
    type Item = Token<'a>;

    fn next(&mut self) -> Option<Self::Item> {
        let pt = self.point;

        if pt.s.is_empty() { return None }

        let mut pm = ParseMaster::new();

        let tmp = pm.alternate()
            .one(|pm| parse_header(pm, pt))
            .one(|pm| parse_inbox(pm, pt))
            .one(|pm| parse_outbox(pm, pt))
            .one(|pm| parse_copy_from(pm, pt))
            .one(|pm| parse_copy_to(pm, pt))
            .one(|pm| parse_bump_up(pm, pt))
            .one(|pm| parse_bump_down(pm, pt))
            .one(|pm| parse_add(pm, pt))
            .one(|pm| parse_sub(pm, pt))
            .one(|pm| parse_label_definition(pm, pt))
            .one(|pm| parse_jump(pm, pt)) // check ordering of jump (prefix-based?)
            .one(|pm| parse_jump_if_zero(pm, pt))
            .one(|pm| parse_jump_if_negative(pm, pt))
            .one(|pm| parse_comment(pm, pt))
            .one(|pm| parse_comment_definition(pm, pt))
            .one(|pm| parse_whitespace(pm, pt))
            .finish();

        match pm.finish(tmp) {
            Progress { status: Status::Success(tok), point } => {
                self.point = point;
                Some(tok)
            },
            Progress { status: Status::Failure(x), point } => {
                println!("Actually an error: {:?}, {:?}", x, point);
                None
            }
        }
    }
}

fn main() {
    let mut f = File::open("36-alphabetizer-both.txt").expect("File?");

    let mut s = String::new();
    f.read_to_string(&mut s).expect("read");

    let t = Thing::new(&s);

    for t in t {
        if let Token::Whitespace(..) = t { continue }
        println!("{:?}", t);
    }
}
<|endoftext|>