<commit_before>// Copyright 2015 Robert Helmer <rhelmer@rhelmer.org>. See the LICENSE
// file at the top-level directory of this distribution.

//! # SymbolAPI
//!
//! SymbolAPI is a microservice which accepts lists of debug symbol names + memory addresses, and
//! returns a list of symbolicated function names.
//!

/// The goal is to take an HTTP JSON request such as:
/// ```
/// {"stacks":[
///   [
///     [0,11723767],
///     [1, 65802]
///    ]
///  ],
///  "memoryMap":[
///    ["xul.pdb","44E4EC8C2F41492B9369D6B9A059577C2"],
///    ["wntdll.pdb","D74F79EB1F8D4A45ABCD2F476CCABACC2"]
///  ],
///  "version":4
/// }
/// ```
/// The symbolapi service then downloads the corresponding symbol files (e.g. from S3), and returns
/// the function names for the corresponding addresses (in the "stacks" array) and returns JSON
/// such as:
/// ```
/// {"symbolicatedStacks": [
///   [
///     "XREMain::XRE_mainRun() (in xul.pdb)",
///     "KiUserCallbackDispatcher (in wntdll.pdb)"]
///   ],
///   "knownModules": [true, true]
/// }
/// ```

extern crate breakpad_symbols;
extern crate flate2;
extern crate hyper;
#[macro_use]
extern crate log;
extern crate log4rs;
extern crate rustc_serialize;
extern crate toml;

use std::collections::HashMap;
use std::env;
use std::fs::{File, create_dir_all};
use std::io::{Read, Write};
use std::path::{PathBuf};
use std::thread;

use breakpad_symbols::{Symbolizer, SimpleSymbolSupplier};
use flate2::read::GzDecoder;
use hyper::Client;
use hyper::header::{ContentEncoding, Encoding};
use hyper::server::{Server, Request, Response};
use hyper::status::StatusCode;
use rustc_serialize::json;

/// Incoming JSON request format.
// required JSON keys are non-snakecase
#[allow(non_snake_case)]
#[derive(RustcDecodable)]
pub struct SymbolRequest {
    pub memoryMap: Vec<(String,String)>,
    // index, offset
    pub stacks: Vec<Vec<(i8,u64)>>,
    pub version: u8,
}

/// Outgoing JSON response format.
// required JSON keys are non-snakecase
#[allow(non_snake_case)]
#[derive(RustcEncodable)]
pub struct SymbolResponse {
    pub symbolicatedStacks: Vec<Vec<String>>,
    pub knownModules: Vec<bool>,
}

fn main() {
    log4rs::init_file("config/log.toml", Default::default()).unwrap();

    let default_port = 5000;
    let port;
    match env::var("PORT") {
        Ok(val) => port = val,
        Err(_) => {
            println!("$PORT unset, using default {}", default_port);
            port = format!("{}", default_port);
        }
    }

    let address = &*format!("0.0.0.0:{}", port);

    info!("Listening on {}", address);
    Server::http(address).unwrap().handle(server).unwrap();
}

/// Stacks come in as a JSON array, but really is a hash map of `<index, address>``
/// where `index` is the position of the `memoryMap` JSON result.
pub fn stacks_to_stack_map(decoded_stacks: Vec<Vec<(i8,u64)>>) -> HashMap<i8, Vec<u64>> {
    debug!("decoded_stacks: {:?}", decoded_stacks);

    let mut stack_map: HashMap<i8, Vec<u64>> = HashMap::new();
    for stack in &decoded_stacks[0] {
        let (index, offset) = *stack;

        let mut offsets = vec!();
        if stack_map.contains_key(&index) {
            offsets = stack_map.get(&index).unwrap().clone();
        }
        offsets.push(offset);
        &stack_map.insert(index, offsets);
    }

    debug!("stack_map: {:?}", stack_map);
    stack_map
}

/// Receives single HTTP requests and demuxes to symbols file fetches from S3 bucket.
pub fn server(mut req: Request, mut res: Response) {
    info!("incoming connection from {}", req.remote_addr);

    match req.method {
        hyper::Post => {
            let mut res = res.start().unwrap();

            let mut buffer = String::new();
            req.read_to_string(&mut buffer).unwrap();
            debug!("raw POST: {:?}", &buffer);

            let decoded: SymbolRequest = json::decode(&buffer).unwrap();

            debug!("decoded memoryMap: {:?}", decoded.memoryMap);
            debug!("decoded stacks: {:?}", decoded.stacks);
            debug!("decoded version: {}", decoded.version);

            let symbol_url = get_config("symbol_urls.public");
            let stack_map = stacks_to_stack_map(decoded.stacks);

            // FIXME limit the number of possible threads
            // TODO maybe push these into a queue and have a thread pool service the queue?
            let symbol_response = json::encode(&client(symbol_url, decoded.memoryMap, stack_map)).unwrap();
            res.write_all(symbol_response.as_bytes()).unwrap();

            res.end().unwrap();
        },
        hyper::Get => {
            let mut res = res.start().unwrap();
            res.write_all("symbolapi, see github.com/rhelmer/symbolapi".as_bytes()).unwrap();
        },
        _ => { *res.status_mut() = StatusCode::MethodNotAllowed },
    }

    debug!("finished serving request");
}

/// Creates multiple client connections to fetch debug symbols from S3 bucket and symbolicate using
/// memory and stack maps, aggregates results and returns SymbolResult.
pub fn client(url: String, memory_map: Vec<(String,String)>, stack_map: HashMap<i8, Vec<u64>>) -> SymbolResponse {
    let mut handles = vec![];
    let mut counter: i8 = 0;
    for (debug_file, debug_id) in memory_map {
        let debug_file_name = debug_file.clone();
        let pdb = debug_file.find(".pdb").unwrap();
        let (symbol_name, _) = debug_file.split_at(pdb);
        let symbol_file = format!("{}.sym", symbol_name);
        let this_url = format!("{}/{}/{}/{}", url, debug_file, debug_id, symbol_file);

        let mut symbol_path = PathBuf::new();
        // TODO get from config
        symbol_path.push("testdata/symbols");

        let mut full_symbol_path = symbol_path.clone();
        full_symbol_path.push(&debug_file);
        full_symbol_path.push(&debug_id);
        full_symbol_path.push(&symbol_file);

        let stack_map_copy = stack_map.clone();

        create_dir_all(&full_symbol_path.parent().unwrap()).unwrap();

        let supplier = SimpleSymbolSupplier::new(vec!(symbol_path.clone()));

        // TODO most of the time is probably spent waiting on I/O, maybe async would be more appropriate?
        // TODO decide min/max possible threads, possibly based on number of cores?
        handles.push(thread::spawn(move || {

            let symbolizer = Symbolizer::new(supplier);
            // FIXME use Arc<Mutex<File>> to prevent concurrent writes
            // TODO only write contents if server version newer
            if !&full_symbol_path.exists() {

                let c = Client::new();
                let mut res = c.get(&this_url).send().unwrap();
                let is_gzipped = match res.headers.get::<ContentEncoding>() {
                    Some(x) => x.contains(&Encoding::Gzip),
                    None => false,
                };

                let mut body = String::new();
                if is_gzipped {
                    let mut raw_body = vec!();
                    res.read_to_end(&mut raw_body).unwrap();
                    let mut d = match GzDecoder::new(&raw_body[..]) {
                        Ok(x) => x,
                        Err(x) => panic!("cannot gunzip {:?}, {:?}", raw_body, x)
                    };
                    d.read_to_string(&mut body).unwrap();
                } else {
                    res.read_to_string(&mut body).unwrap();
                }

                let mut f = File::create(&full_symbol_path).unwrap();
                f.write_all(body.as_bytes()).unwrap();
            }

            debug!("symbol_path: {:?}", &symbol_path);

            let mut symbols = vec!();
            let mut known_module = true;
            for stacks in stack_map_copy.get(&counter) {
                for address in stacks {
                    debug!("attempt to symbolicate: {} for: {:?}", *address, &full_symbol_path);
                    match symbolizer.get_symbol_at_address(&debug_file_name, &debug_id, *address) {
                        Some(x) => symbols.push(x),
                        // return the address rather than function name if symbol not found
                        None => {
                            symbols.push(format!("0x{:x}", address));
                            known_module = false;
                        },
                    }
                }
            }
            debug!("debug_file_name: {:?}, symbols: {:?}", debug_file_name, symbols);

            (debug_file_name, symbols, known_module)
        }));

        counter += 1;
    }

    let mut result = SymbolResponse {
        symbolicatedStacks: vec!(),
        knownModules: vec!(),
    };

    let mut symbolicated_stacks = vec!();

    // pass anything with index -1 through
    for stacks in stack_map.get(&-1) {
        for address in stacks {
            symbolicated_stacks.push(
                format!("0x{:x}", address)
            );
        }
    }

    for handle in handles {
        let (debug_file_name, symbols, known_module) = handle.join().unwrap();

        for symbol in symbols {
            symbolicated_stacks.push(
                format!("{} (in {})", symbol, debug_file_name)
            );
        }
        result.knownModules.push(known_module);
    }

    // the required result format requires this to be a vec-of-vecs
    result.symbolicatedStacks.push(symbolicated_stacks);

    result
}

/// Returns individual values from the configuration file.
fn get_config(value_name: &str) -> String {
    // TODO move to actual file, static str for the moment
    let toml: &'static str = r#"
        [symbol_urls]
        public = "https://s3-us-west-2.amazonaws.com/org.mozilla.crash-stats.symbols-public/v1"
    "#;
    // TODO support multiple URLs
    let value: toml::Value = toml.parse().unwrap();

    value.lookup(value_name).unwrap().as_str().unwrap().to_string()
}
<commit_msg>fix markup in comments<commit_after>// Copyright 2015 Robert Helmer <rhelmer@rhelmer.org>. See the LICENSE
// file at the top-level directory of this distribution.

//! # SymbolAPI
//!
//! SymbolAPI is a microservice which accepts lists of debug symbol names + memory addresses, and
//! returns a list of symbolicated function names.
//!

/// The goal is to take an HTTP JSON request such as:
/// ```
/// {"stacks":[
///   [
///     [0,11723767],
///     [1, 65802]
///    ]
///  ],
///  "memoryMap":[
///    ["xul.pdb","44E4EC8C2F41492B9369D6B9A059577C2"],
///    ["wntdll.pdb","D74F79EB1F8D4A45ABCD2F476CCABACC2"]
///  ],
///  "version":4
/// }
/// ```
/// The symbolapi service then downloads the corresponding symbol files (e.g. from S3), and returns
/// the function names for the corresponding addresses (in the "stacks" array) and returns JSON
/// such as:
/// ```
/// {"symbolicatedStacks": [
///   [
///     "XREMain::XRE_mainRun() (in xul.pdb)",
///     "KiUserCallbackDispatcher (in wntdll.pdb)"]
///   ],
///   "knownModules": [true, true]
/// }
/// ```

extern crate breakpad_symbols;
extern crate flate2;
extern crate hyper;
#[macro_use]
extern crate log;
extern crate log4rs;
extern crate rustc_serialize;
extern crate toml;

use std::collections::HashMap;
use std::env;
use std::fs::{File, create_dir_all};
use std::io::{Read, Write};
use std::path::{PathBuf};
use std::thread;

use breakpad_symbols::{Symbolizer, SimpleSymbolSupplier};
use flate2::read::GzDecoder;
use hyper::Client;
use hyper::header::{ContentEncoding, Encoding};
use hyper::server::{Server, Request, Response};
use hyper::status::StatusCode;
use rustc_serialize::json;

/// Incoming JSON request format.
// required JSON keys are non-snakecase
#[allow(non_snake_case)]
#[derive(RustcDecodable)]
pub struct SymbolRequest {
    pub memoryMap: Vec<(String,String)>,
    // index, offset
    pub stacks: Vec<Vec<(i8,u64)>>,
    pub version: u8,
}

/// Outgoing JSON response format.
// required JSON keys are non-snakecase
#[allow(non_snake_case)]
#[derive(RustcEncodable)]
pub struct SymbolResponse {
    pub symbolicatedStacks: Vec<Vec<String>>,
    pub knownModules: Vec<bool>,
}

fn main() {
    log4rs::init_file("config/log.toml", Default::default()).unwrap();

    let default_port = 5000;
    let port;
    match env::var("PORT") {
        Ok(val) => port = val,
        Err(_) => {
            println!("$PORT unset, using default {}", default_port);
            port = format!("{}", default_port);
        }
    }

    let address = &*format!("0.0.0.0:{}", port);

    info!("Listening on {}", address);
    Server::http(address).unwrap().handle(server).unwrap();
}

/// Stacks come in as a JSON array, but really is a hash map of `<index, address>`
/// where `index` is the position of the `memoryMap` JSON result.
pub fn stacks_to_stack_map(decoded_stacks: Vec<Vec<(i8,u64)>>) -> HashMap<i8, Vec<u64>> {
    debug!("decoded_stacks: {:?}", decoded_stacks);

    let mut stack_map: HashMap<i8, Vec<u64>> = HashMap::new();
    for stack in &decoded_stacks[0] {
        let (index, offset) = *stack;

        let mut offsets = vec!();
        if stack_map.contains_key(&index) {
            offsets = stack_map.get(&index).unwrap().clone();
        }
        offsets.push(offset);
        &stack_map.insert(index, offsets);
    }

    debug!("stack_map: {:?}", stack_map);
    stack_map
}

/// Receives single HTTP requests and demuxes to symbols file fetches from S3 bucket via the
/// client function.
pub fn server(mut req: Request, mut res: Response) {
    info!("incoming connection from {}", req.remote_addr);

    match req.method {
        hyper::Post => {
            let mut res = res.start().unwrap();

            let mut buffer = String::new();
            req.read_to_string(&mut buffer).unwrap();
            debug!("raw POST: {:?}", &buffer);

            let decoded: SymbolRequest = json::decode(&buffer).unwrap();

            debug!("decoded memoryMap: {:?}", decoded.memoryMap);
            debug!("decoded stacks: {:?}", decoded.stacks);
            debug!("decoded version: {}", decoded.version);

            let symbol_url = get_config("symbol_urls.public");
            let stack_map = stacks_to_stack_map(decoded.stacks);

            // FIXME limit the number of possible threads
            // TODO maybe push these into a queue and have a thread pool service the queue?
            let symbol_response = json::encode(&client(symbol_url, decoded.memoryMap, stack_map)).unwrap();
            res.write_all(symbol_response.as_bytes()).unwrap();

            res.end().unwrap();
        },
        hyper::Get => {
            let mut res = res.start().unwrap();
            res.write_all("symbolapi, see github.com/rhelmer/symbolapi".as_bytes()).unwrap();
        },
        _ => { *res.status_mut() = StatusCode::MethodNotAllowed },
    }

    debug!("finished serving request");
}

/// Creates multiple client connections to fetch debug symbols from S3 bucket and symbolicate using
/// memory and stack maps, aggregates results and returns SymbolResult.
pub fn client(url: String, memory_map: Vec<(String,String)>, stack_map: HashMap<i8, Vec<u64>>) -> SymbolResponse {
    let mut handles = vec![];
    let mut counter: i8 = 0;
    for (debug_file, debug_id) in memory_map {
        let debug_file_name = debug_file.clone();
        let pdb = debug_file.find(".pdb").unwrap();
        let (symbol_name, _) = debug_file.split_at(pdb);
        let symbol_file = format!("{}.sym", symbol_name);
        let this_url = format!("{}/{}/{}/{}", url, debug_file, debug_id, symbol_file);

        let mut symbol_path = PathBuf::new();
        // TODO get from config
        symbol_path.push("testdata/symbols");

        let mut full_symbol_path = symbol_path.clone();
        full_symbol_path.push(&debug_file);
        full_symbol_path.push(&debug_id);
        full_symbol_path.push(&symbol_file);

        let stack_map_copy = stack_map.clone();

        create_dir_all(&full_symbol_path.parent().unwrap()).unwrap();

        let supplier = SimpleSymbolSupplier::new(vec!(symbol_path.clone()));

        // TODO most of the time is probably spent waiting on I/O, maybe async would be more appropriate?
        // TODO decide min/max possible threads, possibly based on number of cores?
        handles.push(thread::spawn(move || {

            let symbolizer = Symbolizer::new(supplier);
            // FIXME use Arc<Mutex<File>> to prevent concurrent writes
            // TODO only write contents if server version newer
            if !&full_symbol_path.exists() {

                let c = Client::new();
                let mut res = c.get(&this_url).send().unwrap();
                let is_gzipped = match res.headers.get::<ContentEncoding>() {
                    Some(x) => x.contains(&Encoding::Gzip),
                    None => false,
                };

                let mut body = String::new();
                if is_gzipped {
                    let mut raw_body = vec!();
                    res.read_to_end(&mut raw_body).unwrap();
                    let mut d = match GzDecoder::new(&raw_body[..]) {
                        Ok(x) => x,
                        Err(x) => panic!("cannot gunzip {:?}, {:?}", raw_body, x)
                    };
                    d.read_to_string(&mut body).unwrap();
                } else {
                    res.read_to_string(&mut body).unwrap();
                }

                let mut f = File::create(&full_symbol_path).unwrap();
                f.write_all(body.as_bytes()).unwrap();
            }

            debug!("symbol_path: {:?}", &symbol_path);

            let mut symbols = vec!();
            let mut known_module = true;
            for stacks in stack_map_copy.get(&counter) {
                for address in stacks {
                    debug!("attempt to symbolicate: {} for: {:?}", *address, &full_symbol_path);
                    match symbolizer.get_symbol_at_address(&debug_file_name, &debug_id, *address) {
                        Some(x) => symbols.push(x),
                        // return the address rather than function name if symbol not found
                        None => {
                            symbols.push(format!("0x{:x}", address));
                            known_module = false;
                        },
                    }
                }
            }
            debug!("debug_file_name: {:?}, symbols: {:?}", debug_file_name, symbols);

            (debug_file_name, symbols, known_module)
        }));

        counter += 1;
    }

    let mut result = SymbolResponse {
        symbolicatedStacks: vec!(),
        knownModules: vec!(),
    };

    let mut symbolicated_stacks = vec!();

    // pass anything with index -1 through
    for stacks in stack_map.get(&-1) {
        for address in stacks {
            symbolicated_stacks.push(
                format!("0x{:x}", address)
            );
        }
    }

    for handle in handles {
        let (debug_file_name, symbols, known_module) = handle.join().unwrap();

        for symbol in symbols {
            symbolicated_stacks.push(
                format!("{} (in {})", symbol, debug_file_name)
            );
        }
        result.knownModules.push(known_module);
    }

    // the required result format requires this to be a vec-of-vecs
    result.symbolicatedStacks.push(symbolicated_stacks);

    result
}

/// Returns individual values from the configuration file.
fn get_config(value_name: &str) -> String {
    // TODO move to actual file, static str for the moment
    let toml: &'static str = r#"
        [symbol_urls]
        public = "https://s3-us-west-2.amazonaws.com/org.mozilla.crash-stats.symbols-public/v1"
    "#;
    // TODO support multiple URLs
    let value: toml::Value = toml.parse().unwrap();

    value.lookup(value_name).unwrap().as_str().unwrap().to_string()
}
<|endoftext|>


<commit_before>use std::env;

mod command;

use command::*;

fn main() {
    let args = env::args().skip(1).collect::<Vec<String>>();

    if args.is_empty() {
        print_usage();
        return
    }

    match args[0].as_ref() {
        "help"    => print_usage(),
        "init"    => initialize(),
        "set"     => set_params(),
        "sweep"   => sweep(),
        "start"   => register_with_cron(),
        "end"     => unregister_cron(),
        "destroy" => destroy(),
        _         => print_usage(),
    }
}

fn print_usage() {
    let full_path_to_bin = env::current_exe().unwrap();
    let bin_name         = full_path_to_bin.file_name()
        .unwrap()
        .to_str()
        .unwrap();
    println!(
"Usage:
    {} <command>

Commands:
    help    # Print this message.
    init    # Register current directory.
    start   # Start \"{}\"
    end     # End \"{}\"
    set     # Set parameters.
    sweep   # Sweep files in current directory.
    destroy # Destroy \"{}\"",
        bin_name,
        bin_name,
        bin_name,
        bin_name,
    );
}
<commit_msg>Order commands in help message<commit_after>use std::env;

mod command;

use command::*;

fn main() {
    let args = env::args().skip(1).collect::<Vec<String>>();

    if args.is_empty() {
        print_usage();
        return
    }

    match args[0].as_ref() {
        "help"    => print_usage(),
        "init"    => initialize(),
        "set"     => set_params(),
        "sweep"   => sweep(),
        "start"   => register_with_cron(),
        "end"     => unregister_cron(),
        "destroy" => destroy(),
        _         => print_usage(),
    }
}

fn print_usage() {
    let full_path_to_bin = env::current_exe().unwrap();
    let bin_name         = full_path_to_bin.file_name()
        .unwrap()
        .to_str()
        .unwrap();
    println!(
"Usage:
    {} <command>

Commands:
    help    # Print this message.
    init    # Register current directory.
    set     # Set parameters.
    sweep   # Sweep files in current directory.
    start   # Start \"{}\"
    end     # End \"{}\"
    destroy # Destroy \"{}\"",
        bin_name,
        bin_name,
        bin_name,
        bin_name,
    );
}
<|endoftext|>


<commit_before>extern crate aqua;
extern crate aqua_web;
extern crate conduit_hyper;
extern crate dotenv;
extern crate env_logger;

use aqua::{controllers, util};
use aqua_web::{mw, plug};
use conduit_hyper::Server;
use dotenv::dotenv;

fn main() {
    // load configuration from .env
    dotenv().expect("must provide .env file, see README (TODO: haha jk)");
    env_logger::init().expect("could not initialize console logging");

    // these are application extensions which our controllers expect to be present
    let extensions = plug::Pipeline::new()
        .then(util::db::DbMiddleware::new())
        .then(util::template::TemplateMiddleware::new());

    // the main entry point into our application
    let router = mw::Router::new()
        .get("/dash",                 controllers::dash::index)
        .get("/tags/{schema}/{name}", controllers::dash::show_tags)
        .get("/entries/{id}",         controllers::entries::show_id)
        .get("/entries/thumb/{id}",   controllers::entries::thumb_id)
        .post("/entries/upload",      controllers::entries::submit);

    // the endpoint provides basic HTTP massaging before our router is invoked
    // with the current request data ...
    let endpoint = plug::Pipeline::new()
        .then(util::timer::plug)
        .then(util::try_file::TryFileMiddleware)
        .then(mw::MultipartParser)
        .then(extensions)
        .then(router);

    Server::http("0.0.0.0:3000")
        .expect("could not start http server")
        .handle(endpoint)
        .expect("could not attach handler");
}
<commit_msg>(git) Oops! Forgot to check in renamed routing table.<commit_after>extern crate aqua;
extern crate aqua_web;
extern crate conduit_hyper;
extern crate dotenv;
extern crate env_logger;

use aqua::{controllers, util};
use aqua_web::{mw, plug};
use conduit_hyper::Server;
use dotenv::dotenv;

fn main() {
    // load configuration from .env
    dotenv().expect("must provide .env file, see README (TODO: haha jk)");
    env_logger::init().expect("could not initialize console logging");

    // these are application extensions which our controllers expect to be present
    let extensions = plug::Pipeline::new()
        .then(util::db::DbMiddleware::new())
        .then(util::template::TemplateMiddleware::new());

    // the main entry point into our application
    let router = mw::Router::new()
        .get("/dash",                 controllers::dash::index)
        .get("/tags/{schema}/{name}", controllers::dash::show_tags)
        .get("/entries/{id}",         controllers::entries::show)
        .get("/entries/thumb/{id}",   controllers::entries::show_thumb)
        .post("/entries/upload",      controllers::entries::submit);

    // the endpoint provides basic HTTP massaging before our router is invoked
    // with the current request data ...
    let endpoint = plug::Pipeline::new()
        .then(util::timer::plug)
        .then(util::try_file::TryFileMiddleware)
        .then(mw::MultipartParser)
        .then(extensions)
        .then(router);

    Server::http("0.0.0.0:3000")
        .expect("could not start http server")
        .handle(endpoint)
        .expect("could not attach handler");
}
<|endoftext|>


<commit_before>use std::rc::Rc;

type Error = &'static str;

#[derive(Debug,Clone)]
enum Code {
    Integer(i64),
    Rational(i64, i64),
    String(String),
    Add(Rc<Code>, Rc<Code>),
    Sub(Rc<Code>, Rc<Code>),
    Div(Rc<Code>, Rc<Code>),
    Mul(Rc<Code>, Rc<Code>),
    Neg(Rc<Code>),
}

// Calculate the greatest common divisor using Euclid's algorithm.
fn gcd(mut a: i64, mut b: i64) -> i64 {
    let mut c;
    while a != 0 {
        c = a;
        a = b % a;
        b = c;
    }
    b
}

// This normalizes inputs and sometimes even converts to integers.
fn new_rational(mut n: i64, mut d: i64) -> Code {
    let g = gcd(n, d);
    n = n / g;
    d = d / g;
    if d < 0 {
      n = -n;
      d = -d;
    }
    if d == 1 {
        Code::Integer(n)
    }
    else {
        Code::Rational(n, d)
    }
}

fn rational_add(n1: i64, d1: i64, n2: i64, d2: i64) -> Code {
  if d1 == d2 {
    // Fast path for common divisors.
    new_rational(n1 + n2, d1)
  }
  else {
    let g = gcd(d1, d2);
    new_rational(n1 * (d2 / g) + n2 * (d1 / g), d1 / g * d2)
  }

}

fn add(left: &Code, right: &Code) -> Result<Code, Error> {
    match (try!(eval(left)), try!(eval(right))) {
        (Code::Integer(a), Code::Integer(b)) => Ok(Code::Integer(a + b)),
        (Code::Rational(n, d), Code::Integer(b)) => Ok(rational_add(n, d, b, 1)),
        (Code::Integer(a), Code::Rational(n, d)) => Ok(rational_add(a, 1, n, d)),
        (Code::Rational(n1, d1), Code::Rational(n2, d2)) => Ok(rational_add(n1, d1, n2, d2)),
        _ => Err("Add requires two numbers"),
    }
}

fn subtract(left: &Code, right: &Code) -> Result<Code, Error> {
    match (try!(eval(left)), try!(eval(right))) {
        (Code::Integer(a), Code::Integer(b)) => Ok(Code::Integer(a - b)),
        (Code::Rational(n, d), Code::Integer(b)) => Ok(rational_add(n, d, -b, 1)),
        (Code::Integer(a), Code::Rational(n, d)) => Ok(rational_add(a, 1, -n, d)),
        (Code::Rational(n1, d1), Code::Rational(n2, d2)) => Ok(rational_add(n1, d1, -n2, d2)),
        _ => Err("Subtract requires two numbers"),
    }
}

fn multiply(left: &Code, right: &Code) -> Result<Code, Error> {
    match (try!(eval(left)), try!(eval(right))) {
        (Code::Integer(a), Code::Integer(b)) => Ok(Code::Integer(a * b)),
        (Code::Rational(n, d), Code::Integer(b)) => Ok(new_rational(n * b, d)),
        (Code::Integer(a), Code::Rational(n, d)) => Ok(new_rational(a * n, d)),
        (Code::Rational(n1, d1), Code::Rational(n2, d2)) => Ok(new_rational(n1 * n2, d1 * d2)),
        _ => Err("Multiply requires two numbers"),
    }
}

fn divide(left: &Code, right: &Code) -> Result<Code, Error> {
    match (try!(eval(left)), try!(eval(right))) {
        (Code::Integer(a), Code::Integer(b)) => Ok(new_rational(a, b)),
        (Code::Rational(n, d), Code::Integer(b)) => Ok(new_rational(n, d * b)),
        (Code::Integer(a), Code::Rational(n, d)) => Ok(new_rational(a * d, n)),
        (Code::Rational(n1, d1), Code::Rational(n2, d2)) => Ok(new_rational(n1 * d2, d1 * n2)),
        _ => Err("Divide requires two numbers"),
    }
}

fn negate(num: &Code) -> Result<Code, Error> {
    match num {
        &Code::Integer(a) => Ok(Code::Integer(-a)),
        &Code::Rational(n, d) => Ok(Code::Rational(-n, d)),
        _ => Err("Negate requires a number"),
    }
}

fn eval(expr: &Code) -> Result<Code, Error> {
    match expr {
        &Code::Add(ref left, ref right) => add(left, right),
        &Code::Sub(ref left, ref right) => subtract(left, right),
        &Code::Mul(ref left, ref right) => multiply(left, right),
        &Code::Div(ref left, ref right) => divide(left, right),
        &Code::Neg(ref num) => negate(num),
        n @ &Code::Integer(_) => Ok(n.clone()),
        n @ &Code::Rational(_, _) => Ok(n.clone()),
        n @ &Code::String(_) => Ok(n.clone()),
    }
}

fn test(left: Code, right: Code) {
    let a = Rc::new(left);
    let b = Rc::new(right);
    let mut e = Rc::new(Code::Add(a.clone(), b.clone()));
    println!("{:?} = {:?}", e, eval(&e));
    e = Rc::new(Code::Sub(a.clone(), b.clone()));
    println!("{:?} = {:?}", e, eval(&e));
    e = Rc::new(Code::Mul(a.clone(), b.clone()));
    println!("{:?} = {:?}", e, eval(&e));
    e = Rc::new(Code::Div(a.clone(), b.clone()));
    println!("{:?} = {:?}", e, eval(&e));
    e = Rc::new(Code::Neg(a.clone()));
    println!("{:?} = {:?}", e, eval(&e));
}

fn main() {
    test(Code::Integer(44), Code::Integer(14));
    test(Code::Integer(14), Code::Integer(44));
    test(Code::Integer(44), Code::Integer(-14));
    test(Code::Integer(14), Code::Integer(-44));
    test(Code::Integer(-44), Code::Integer(14));
    test(Code::Integer(-14), Code::Integer(44));
    test(Code::Integer(-44), Code::Integer(-14));
    test(Code::Integer(-14), Code::Integer(-44));
    test(new_rational(1, 2), new_rational(2, 1));
    test(new_rational(2, 1), new_rational(1, 2));
    test(new_rational(1, 2), new_rational(1, 2));
    test(new_rational(2, 2), new_rational(2, 1));
    test(Code::String(str::to_string("Input string")),
         new_rational(2, 1));
    let e = Code::Div(
        Rc::new(Code::Add(
            Rc::new(Code::Rational(1, 2)),
            Rc::new(Code::Rational(1, 3)),
        )),
        Rc::new(Code::Sub(
            Rc::new(Code::Integer(2)),
            Rc::new(Code::Integer(5)),
        )),
    );
    println!("{:?} = {:?}", e, eval(&e));

}
<commit_msg>Implement basic fmt<commit_after>use std::fmt;
use std::rc::Rc;

type Error = &'static str;

#[derive(Debug,Clone)]
enum Code {
    Integer(i64),
    Rational(i64, i64),
    String(String),
    Add(Rc<Code>, Rc<Code>),
    Sub(Rc<Code>, Rc<Code>),
    Div(Rc<Code>, Rc<Code>),
    Mul(Rc<Code>, Rc<Code>),
    Neg(Rc<Code>),
}

impl fmt::Display for Code {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &Code::Integer(n) => write!(f, "{}", n),
            &Code::Rational(n, d) => write!(f, "{}/{}", n, d),
            &Code::String(ref s) => write!(f, "'{}'", s),
            &Code::Add(ref left, ref right) => write!(f, "({} + {})", left, right),
            &Code::Sub(ref left, ref right) => write!(f, "({} - {})", left, right),
            &Code::Div(ref left, ref right) => write!(f, "({} / {})", left, right),
            &Code::Mul(ref left, ref right) => write!(f, "({} * {})", left, right),
            &Code::Neg(ref num) => write!(f, "-({})", num),
        }
    }
}

// Calculate the greatest common divisor using Euclid's algorithm.
fn gcd(mut a: i64, mut b: i64) -> i64 {
    let mut c;
    while a != 0 {
        c = a;
        a = b % a;
        b = c;
    }
    b
}

// This normalizes inputs and sometimes even converts to integers.
fn new_rational(mut n: i64, mut d: i64) -> Code {
    let g = gcd(n, d);
    n = n / g;
    d = d / g;
    if d < 0 {
      n = -n;
      d = -d;
    }
    if d == 1 {
        Code::Integer(n)
    }
    else {
        Code::Rational(n, d)
    }
}

fn rational_add(n1: i64, d1: i64, n2: i64, d2: i64) -> Code {
  if d1 == d2 {
    // Fast path for common divisors.
    new_rational(n1 + n2, d1)
  }
  else {
    let g = gcd(d1, d2);
    new_rational(n1 * (d2 / g) + n2 * (d1 / g), d1 / g * d2)
  }

}

fn add(left: &Code, right: &Code) -> Result<Code, Error> {
    match (try!(eval(left)), try!(eval(right))) {
        (Code::Integer(a), Code::Integer(b)) => Ok(Code::Integer(a + b)),
        (Code::Rational(n, d), Code::Integer(b)) => Ok(rational_add(n, d, b, 1)),
        (Code::Integer(a), Code::Rational(n, d)) => Ok(rational_add(a, 1, n, d)),
        (Code::Rational(n1, d1), Code::Rational(n2, d2)) => Ok(rational_add(n1, d1, n2, d2)),
        _ => Err("Add requires two numbers"),
    }
}

fn subtract(left: &Code, right: &Code) -> Result<Code, Error> {
    match (try!(eval(left)), try!(eval(right))) {
        (Code::Integer(a), Code::Integer(b)) => Ok(Code::Integer(a - b)),
        (Code::Rational(n, d), Code::Integer(b)) => Ok(rational_add(n, d, -b, 1)),
        (Code::Integer(a), Code::Rational(n, d)) => Ok(rational_add(a, 1, -n, d)),
        (Code::Rational(n1, d1), Code::Rational(n2, d2)) => Ok(rational_add(n1, d1, -n2, d2)),
        _ => Err("Subtract requires two numbers"),
    }
}

fn multiply(left: &Code, right: &Code) -> Result<Code, Error> {
    match (try!(eval(left)), try!(eval(right))) {
        (Code::Integer(a), Code::Integer(b)) => Ok(Code::Integer(a * b)),
        (Code::Rational(n, d), Code::Integer(b)) => Ok(new_rational(n * b, d)),
        (Code::Integer(a), Code::Rational(n, d)) => Ok(new_rational(a * n, d)),
        (Code::Rational(n1, d1), Code::Rational(n2, d2)) => Ok(new_rational(n1 * n2, d1 * d2)),
        _ => Err("Multiply requires two numbers"),
    }
}

fn divide(left: &Code, right: &Code) -> Result<Code, Error> {
    match (try!(eval(left)), try!(eval(right))) {
        (Code::Integer(a), Code::Integer(b)) => Ok(new_rational(a, b)),
        (Code::Rational(n, d), Code::Integer(b)) => Ok(new_rational(n, d * b)),
        (Code::Integer(a), Code::Rational(n, d)) => Ok(new_rational(a * d, n)),
        (Code::Rational(n1, d1), Code::Rational(n2, d2)) => Ok(new_rational(n1 * d2, d1 * n2)),
        _ => Err("Divide requires two numbers"),
    }
}

fn negate(num: &Code) -> Result<Code, Error> {
    match num {
        &Code::Integer(a) => Ok(Code::Integer(-a)),
        &Code::Rational(n, d) => Ok(Code::Rational(-n, d)),
        _ => Err("Negate requires a number"),
    }
}

fn eval(expr: &Code) -> Result<Code, Error> {
    match expr {
        &Code::Add(ref left, ref right) => add(left, right),
        &Code::Sub(ref left, ref right) => subtract(left, right),
        &Code::Mul(ref left, ref right) => multiply(left, right),
        &Code::Div(ref left, ref right) => divide(left, right),
        &Code::Neg(ref num) => negate(num),
        n @ &Code::Integer(_) => Ok(n.clone()),
        n @ &Code::Rational(_, _) => Ok(n.clone()),
        n @ &Code::String(_) => Ok(n.clone()),
    }
}

fn test(left: Code, right: Code) {
    let a = Rc::new(left);
    let b = Rc::new(right);
    let mut e = Rc::new(Code::Add(a.clone(), b.clone()));
    println!("{} = {:?}", e, eval(&e));
    e = Rc::new(Code::Sub(a.clone(), b.clone()));
    println!("{} = {:?}", e, eval(&e));
    e = Rc::new(Code::Mul(a.clone(), b.clone()));
    println!("{} = {:?}", e, eval(&e));
    e = Rc::new(Code::Div(a.clone(), b.clone()));
    println!("{} = {:?}", e, eval(&e));
    e = Rc::new(Code::Neg(a.clone()));
    println!("{} = {:?}", e, eval(&e));
}

fn main() {
    test(Code::Integer(44), Code::Integer(14));
    test(Code::Integer(14), Code::Integer(44));
    test(Code::Integer(44), Code::Integer(-14));
    test(Code::Integer(14), Code::Integer(-44));
    test(Code::Integer(-44), Code::Integer(14));
    test(Code::Integer(-14), Code::Integer(44));
    test(Code::Integer(-44), Code::Integer(-14));
    test(Code::Integer(-14), Code::Integer(-44));
    test(new_rational(1, 2), new_rational(2, 1));
    test(new_rational(2, 1), new_rational(1, 2));
    test(new_rational(1, 2), new_rational(1, 2));
    test(new_rational(2, 2), new_rational(2, 1));
    test(Code::String(str::to_string("Input string")),
         new_rational(2, 1));
    let e = Code::Div(
        Rc::new(Code::Add(
            Rc::new(Code::Rational(1, 2)),
            Rc::new(Code::Rational(1, 3)),
        )),
        Rc::new(Code::Sub(
            Rc::new(Code::Integer(2)),
            Rc::new(Code::Integer(5)),
        )),
    );
    println!("{} = {:?}", e, eval(&e));

}
<|endoftext|>


<commit_before>#[macro_use]
extern crate nom;

use std::io::stdin;
use std::io::Read;
use std::io;
use std::fmt::{self, Display, Debug};
use std::ffi::{CStr, FromBytesWithNulError};
//use std::io::BufRead;
//use nom::{Producer, Move, Input, Consumer, ConsumerState};
use nom::{le_u32};

/*
pub struct BufReadProducer<R: Read + BufRead> {
    reader: R
}

impl<R: Read + BufRead> BufReadProducer<R> {
    pub fn new(reader: R) -> BufReadProducer<R> {
        BufReadProducer { reader: reader }
    }
}

impl<'x, R: Read + BufRead> Producer<'x, &'x [u8], Move> for BufReadProducer<R> {
    fn apply<'a,O,E>(&'x mut self, consumer: &'a mut Consumer<&'x[u8],O,E,Move>) -> &'a ConsumerState<O,E,Move> {
        match consumer.state() {
            &ConsumerState::Continue(ref m) => {
                println!("{:?}", m);
                match *m {
                    Move::Await(_) => (),
                    Move::Consume(bytes) => self.reader.consume(bytes),
                    Move::Seek(position) => panic!("Can't seek BufReadProducer: {:?}", position)
                }

            },
            _  => return consumer.state()
        }

        match self.reader.fill_buf() {
            //TODO: it will probably not give any more data if buf is full -
            //infinite loop!?
            Ok(ref data) => consumer.handle(Input::Element(data)),
            Err(_) => consumer.handle(Input::Eof(None))
        }
    }

    fn run<'a: 'x,O,E: 'x>(&'x mut self, consumer: &'a mut Consumer<&'x[u8],O,E,Move>)   -> Option<&O> {
        //TODO: keep calling apply until we get Done or Err
        //TODO: handle Eof
        if let &ConsumerState::Done(_,ref o) = self.apply(consumer) {
            Some(o)
        } else {
            None
        }
    }
}

consumer_from_parser!(VslConsumer<VslRecordHeader>, vsl_record_header);
consumer_from_parser!(VslTagConsumer<()>, vsl_tag);
*/

/*
 * Shared memory log format
 *
 * The log member points to an array of 32bit unsigned integers containing
 * log records.
 *
 * Each logrecord consist of:
 *    [n]               = ((type & 0xff) << 24) | (length & 0xffff)
 *    [n + 1]           = ((marker & 0x03) << 30) | (identifier & 0x3fffffff)
 *    [n + 2] ... [m]   = content (NUL-terminated)
 *
 * Logrecords are NUL-terminated so that string functions can be run
 * directly on the shmlog data.
 *
 * Notice that the constants in these macros cannot be changed without
 * changing corresponding magic numbers in varnishd/cache/cache_shmlog.c
 */

const VSL_LENOFFSET: u32 = 24;
const VSL_LENMASK: u32 = 0xffff;
const VSL_MARKERMASK: u32 = 0x03;
//const VSL_CLIENTMARKER: u32 = 1 << 30;
//const VSL_BACKENDMARKER: u32 = 1 << 31;
const VSL_IDENTOFFSET: u32 = 30;
const VSL_IDENTMASK: u32 = !(3 << VSL_IDENTOFFSET);

#[derive(Debug)]
enum VslTag {
    BinaryVsl
}

named!(vsl_tag<&[u8], VslTag>, chain!(
        tag!(b"VSL\0"),
        || {
            VslTag::BinaryVsl
        }));

#[derive(Debug)]
struct VslRecordHeader {
    pub tag: u8,
    pub len: u16,
    pub marker: u8,
    pub ident: u32,
}

fn vsl_record_header<'b>(input: &'b[u8]) -> nom::IResult<&'b[u8], VslRecordHeader, u32> {
    chain!(
        input, r1: le_u32 ~ r2: le_u32,
        || {
            VslRecordHeader {
                tag: (r1 >> VSL_LENOFFSET) as u8,
                len: (r1 & VSL_LENMASK) as u16,
                marker: (r2 & VSL_MARKERMASK >> VSL_IDENTOFFSET) as u8,
                ident: r2 & VSL_IDENTMASK,
            }
        })
}

struct VslRecord<'b> {
    pub tag: u8,
    pub marker: u8,
    pub ident: u32,
    pub data: &'b[u8],
}

impl<'b> VslRecord<'b> {
    fn body(&'b self) -> Result<&'b CStr, FromBytesWithNulError> {
        CStr::from_bytes_with_nul(self.data)
    }
}

impl<'b> Debug for VslRecord<'b> {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        f.debug_struct("VSL Record")
            .field("tag", &self.tag)
            .field("marker", &self.marker)
            .field("ident", &self.ident)
            .field("body", &self.body())
            .finish()
    }
}

fn is_zero(i: u8) -> bool { i == 0 }

fn vsl_record<'b>(input: &'b[u8]) -> nom::IResult<&'b[u8], VslRecord<'b>, u32> {
    chain!(
        input,
        header: vsl_record_header ~ data: take!(header.len) ~ take_while!(is_zero),
        || {
            VslRecord { tag: header.tag, marker: header.marker, ident: header.ident, data: data }
        })
}

fn binary_vsl_records<'b>(input: &'b[u8]) -> nom::IResult<&'b[u8], Vec<VslRecord<'b>>, u32> {
    chain!(
        input, vsl_tag ~ records: many1!(vsl_record),
        || { records })
}

#[derive(Debug)]
enum VslError<I, E> {
    IO(io::Error),
    NomErr(nom::Err<I, E>),
    NomNeeded(nom::Needed)
}

impl<I, E> From<io::Error> for VslError<I, E> {
    fn from(e: io::Error) -> VslError<I, E> {
        VslError::IO(e)
    }
}

impl<I, E> From<nom::Err<I, E>> for VslError<I, E> {
    fn from(e: nom::Err<I, E>) -> VslError<I, E> {
        VslError::NomErr(e)
    }
}

impl<I, E> From<nom::Needed> for VslError<I, E> {
    fn from(e: nom::Needed) -> VslError<I, E> {
        VslError::NomNeeded(e)
    }
}

trait ToVslResult<I, O, E> {
    fn into_vsl_result(self) -> Result<O, VslError<I, E>>;
}

impl<I, O, E> ToVslResult<I, O, E> for nom::IResult<I, O, E> {
    fn into_vsl_result(self) -> Result<O, VslError<I, E>> {
        match self {
            nom::IResult::Done(_, out) => Ok(out),
            nom::IResult::Error(e) => Err(From::from(e)),
            nom::IResult::Incomplete(n) => Err(From::from(n)),
        }
    }
}

impl<I, E> Display for VslError<I, E> where I: Debug, E: Debug {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &VslError::IO(ref e) => write!(f, "Failed to read VSL data: {}", e),
            &VslError::NomErr(ref e) => write!(f, "Failed to parse VSL data: {}", e),
            &VslError::NomNeeded(ref e) => write!(f, "Not enought data to parse VSL: {:?}", e),
        }
    }
}

fn main() {
    let stdin = stdin();
    let mut stdin = stdin.lock();

    let mut buf = Vec::new();
    stdin.read_to_end(&mut buf).unwrap();

    let records = binary_vsl_records(buf.as_slice()).into_vsl_result().unwrap();
    for record in records {
        println!("{:?}", &record);
    }
}
<commit_msg>cleanup<commit_after>#[macro_use]
extern crate nom;

use std::io::stdin;
use std::io::Read;
use std::io;
use std::fmt::{self, Display, Debug};
use std::ffi::{CStr, FromBytesWithNulError};
//use std::io::BufRead;
//use nom::{Producer, Move, Input, Consumer, ConsumerState};
use nom::{le_u32};

/*
pub struct BufReadProducer<R: Read + BufRead> {
    reader: R
}

impl<R: Read + BufRead> BufReadProducer<R> {
    pub fn new(reader: R) -> BufReadProducer<R> {
        BufReadProducer { reader: reader }
    }
}

impl<'x, R: Read + BufRead> Producer<'x, &'x [u8], Move> for BufReadProducer<R> {
    fn apply<'a,O,E>(&'x mut self, consumer: &'a mut Consumer<&'x[u8],O,E,Move>) -> &'a ConsumerState<O,E,Move> {
        match consumer.state() {
            &ConsumerState::Continue(ref m) => {
                println!("{:?}", m);
                match *m {
                    Move::Await(_) => (),
                    Move::Consume(bytes) => self.reader.consume(bytes),
                    Move::Seek(position) => panic!("Can't seek BufReadProducer: {:?}", position)
                }

            },
            _  => return consumer.state()
        }

        match self.reader.fill_buf() {
            //TODO: it will probably not give any more data if buf is full -
            //infinite loop!?
            Ok(ref data) => consumer.handle(Input::Element(data)),
            Err(_) => consumer.handle(Input::Eof(None))
        }
    }

    fn run<'a: 'x,O,E: 'x>(&'x mut self, consumer: &'a mut Consumer<&'x[u8],O,E,Move>)   -> Option<&O> {
        //TODO: keep calling apply until we get Done or Err
        //TODO: handle Eof
        if let &ConsumerState::Done(_,ref o) = self.apply(consumer) {
            Some(o)
        } else {
            None
        }
    }
}

consumer_from_parser!(VslConsumer<VslRecordHeader>, vsl_record_header);
consumer_from_parser!(VslTagConsumer<()>, vsl_tag);
*/

/*
 * Shared memory log format
 *
 * The log member points to an array of 32bit unsigned integers containing
 * log records.
 *
 * Each logrecord consist of:
 *    [n]               = ((type & 0xff) << 24) | (length & 0xffff)
 *    [n + 1]           = ((marker & 0x03) << 30) | (identifier & 0x3fffffff)
 *    [n + 2] ... [m]   = content (NUL-terminated)
 *
 * Logrecords are NUL-terminated so that string functions can be run
 * directly on the shmlog data.
 *
 * Notice that the constants in these macros cannot be changed without
 * changing corresponding magic numbers in varnishd/cache/cache_shmlog.c
 */

const VSL_LENOFFSET: u32 = 24;
const VSL_LENMASK: u32 = 0xffff;
const VSL_MARKERMASK: u32 = 0x03;
//const VSL_CLIENTMARKER: u32 = 1 << 30;
//const VSL_BACKENDMARKER: u32 = 1 << 31;
const VSL_IDENTOFFSET: u32 = 30;
const VSL_IDENTMASK: u32 = !(3 << VSL_IDENTOFFSET);

named!(binary_vsl_tag<&[u8], &[u8]>, tag!(b"VSL\0"));

#[derive(Debug)]
struct VslRecordHeader {
    pub tag: u8,
    pub len: u16,
    pub marker: u8,
    pub ident: u32,
}

fn vsl_record_header<'b>(input: &'b[u8]) -> nom::IResult<&'b[u8], VslRecordHeader, u32> {
    chain!(
        input, r1: le_u32 ~ r2: le_u32,
        || {
            VslRecordHeader {
                tag: (r1 >> VSL_LENOFFSET) as u8,
                len: (r1 & VSL_LENMASK) as u16,
                marker: (r2 & VSL_MARKERMASK >> VSL_IDENTOFFSET) as u8,
                ident: r2 & VSL_IDENTMASK,
            }
        })
}

struct VslRecord<'b> {
    pub tag: u8,
    pub marker: u8,
    pub ident: u32,
    pub data: &'b[u8],
}

impl<'b> VslRecord<'b> {
    fn body(&'b self) -> Result<&'b CStr, FromBytesWithNulError> {
        CStr::from_bytes_with_nul(self.data)
    }
}

impl<'b> Debug for VslRecord<'b> {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        f.debug_struct("VSL Record")
            .field("tag", &self.tag)
            .field("marker", &self.marker)
            .field("ident", &self.ident)
            .field("body", &self.body())
            .finish()
    }
}

fn is_zero(i: u8) -> bool { i == 0 }

fn vsl_record<'b>(input: &'b[u8]) -> nom::IResult<&'b[u8], VslRecord<'b>, u32> {
    chain!(
        input,
        header: vsl_record_header ~ data: take!(header.len) ~ take_while!(is_zero),
        || {
            VslRecord { tag: header.tag, marker: header.marker, ident: header.ident, data: data }
        })
}

fn binary_vsl_records<'b>(input: &'b[u8]) -> nom::IResult<&'b[u8], Vec<VslRecord<'b>>, u32> {
    chain!(
        input, binary_vsl_tag ~ records: many1!(vsl_record),
        || { records })
}

#[derive(Debug)]
enum VslError<I, E> {
    IO(io::Error),
    NomErr(nom::Err<I, E>),
    NomNeeded(nom::Needed)
}

impl<I, E> From<io::Error> for VslError<I, E> {
    fn from(e: io::Error) -> VslError<I, E> {
        VslError::IO(e)
    }
}

impl<I, E> From<nom::Err<I, E>> for VslError<I, E> {
    fn from(e: nom::Err<I, E>) -> VslError<I, E> {
        VslError::NomErr(e)
    }
}

impl<I, E> From<nom::Needed> for VslError<I, E> {
    fn from(e: nom::Needed) -> VslError<I, E> {
        VslError::NomNeeded(e)
    }
}

trait ToVslResult<I, O, E> {
    fn into_vsl_result(self) -> Result<O, VslError<I, E>>;
}

impl<I, O, E> ToVslResult<I, O, E> for nom::IResult<I, O, E> {
    fn into_vsl_result(self) -> Result<O, VslError<I, E>> {
        match self {
            nom::IResult::Done(_, out) => Ok(out),
            nom::IResult::Error(e) => Err(From::from(e)),
            nom::IResult::Incomplete(n) => Err(From::from(n)),
        }
    }
}

impl<I, E> Display for VslError<I, E> where I: Debug, E: Debug {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &VslError::IO(ref e) => write!(f, "Failed to read VSL data: {}", e),
            &VslError::NomErr(ref e) => write!(f, "Failed to parse VSL data: {}", e),
            &VslError::NomNeeded(ref e) => write!(f, "Not enought data to parse VSL: {:?}", e),
        }
    }
}

fn main() {
    let stdin = stdin();
    let mut stdin = stdin.lock();

    let mut buf = Vec::new();
    stdin.read_to_end(&mut buf).unwrap();

    let records = binary_vsl_records(buf.as_slice()).into_vsl_result().unwrap();
    for record in records {
        println!("{:?}", &record);
    }
}
<|endoftext|>


<commit_before>extern crate byteorder;
extern crate chan;
extern crate csv;
extern crate csv_index;
extern crate docopt;
extern crate filetime;
extern crate num_cpus;
extern crate rand;
extern crate regex;
extern crate serde;
#[macro_use]
extern crate serde_derive;
extern crate stats;
extern crate tabwriter;
extern crate threadpool;

use std::borrow::ToOwned;
use std::env;
use std::fmt;
use std::io;
use std::process;

use docopt::Docopt;

macro_rules! wout {
    ($($arg:tt)*) => ({
        use std::io::Write;
        (writeln!(&mut ::std::io::stdout(), $($arg)*)).unwrap();
    });
}

macro_rules! werr {
    ($($arg:tt)*) => ({
        use std::io::Write;
        (writeln!(&mut ::std::io::stderr(), $($arg)*)).unwrap();
    });
}

macro_rules! fail {
    ($e:expr) => (Err(::std::convert::From::from($e)));
}

macro_rules! command_list {
    () => (
"
    cat         Concatenate by row or column
    count       Count records
    fixlengths  Makes all records have same length
    flatten     Show one field per line
    fmt         Format CSV output (change field delimiter)
    frequency   Show frequency tables
    headers     Show header names
    help        Show this usage message.
    index       Create CSV index for faster access
    input       Read CSV data with special quoting rules
    join        Join CSV files
    sample      Randomly sample CSV data
    search      Search CSV data with regexes
    select      Select columns from CSV
    slice       Slice records from CSV
    sort        Sort CSV data
    split       Split CSV data into many files
    stats       Compute basic statistics
    table       Align CSV data into columns
"
    )
}

mod cmd;
mod config;
mod index;
mod select;
mod util;

static USAGE: &'static str = concat!("
Usage:
    xsv <command> [<args>...]
    xsv [options]

Options:
    --list        List all commands available.
    -h, --help    Display this message
    <command> -h  Display the command help message
    --version     Print version info and exit

Commands:", command_list!());

#[derive(Deserialize)]
struct Args {
    arg_command: Option<Command>,
    flag_list: bool,
}

fn main() {
    let args: Args = Docopt::new(USAGE)
                            .and_then(|d| d.options_first(true)
                                           .version(Some(util::version()))
                                           .deserialize())
                            .unwrap_or_else(|e| e.exit());
    if args.flag_list {
        wout!(concat!("Installed commands:", command_list!()));
        return;
    }
    match args.arg_command {
        None => {
            werr!(concat!(
                "xsv is a suite of CSV command line utilities.

Please choose one of the following commands:",
                command_list!()));
            process::exit(0);
        }
        Some(cmd) => {
            match cmd.run() {
                Ok(()) => process::exit(0),
                Err(CliError::Flag(err)) => err.exit(),
                Err(CliError::Csv(err)) => {
                    werr!("{}", err);
                    process::exit(1);
                }
                Err(CliError::Io(ref err))
                        if err.kind() == io::ErrorKind::BrokenPipe => {
                    process::exit(0);
                }
                Err(CliError::Io(err)) => {
                    werr!("{}", err);
                    process::exit(1);
                }
                Err(CliError::Other(msg)) => {
                    werr!("{}", msg);
                    process::exit(1);
                }
            }
        }
    }
}

#[derive(Debug, Deserialize)]
enum Command {
    Cat,
    Count,
    FixLengths,
    Flatten,
    Fmt,
    Frequency,
    Headers,
    Help,
    Index,
    Input,
    Join,
    Partition,
    Sample,
    Search,
    Select,
    Slice,
    Sort,
    Split,
    Stats,
    Table,
}

impl Command {
    fn run(self) -> CliResult<()> {
        let argv: Vec<_> = env::args().map(|v| v.to_owned()).collect();
        let argv: Vec<_> = argv.iter().map(|s| &**s).collect();
        let argv = &*argv;
        match self {
            Command::Cat => cmd::cat::run(argv),
            Command::Count => cmd::count::run(argv),
            Command::FixLengths => cmd::fixlengths::run(argv),
            Command::Flatten => cmd::flatten::run(argv),
            Command::Fmt => cmd::fmt::run(argv),
            Command::Frequency => cmd::frequency::run(argv),
            Command::Headers => cmd::headers::run(argv),
            Command::Help => { wout!("{}", USAGE); Ok(()) }
            Command::Index => cmd::index::run(argv),
            Command::Input => cmd::input::run(argv),
            Command::Join => cmd::join::run(argv),
            Command::Partition => cmd::partition::run(argv),
            Command::Sample => cmd::sample::run(argv),
            Command::Search => cmd::search::run(argv),
            Command::Select => cmd::select::run(argv),
            Command::Slice => cmd::slice::run(argv),
            Command::Sort => cmd::sort::run(argv),
            Command::Split => cmd::split::run(argv),
            Command::Stats => cmd::stats::run(argv),
            Command::Table => cmd::table::run(argv),
        }
    }
}

pub type CliResult<T> = Result<T, CliError>;

#[derive(Debug)]
pub enum CliError {
    Flag(docopt::Error),
    Csv(csv::Error),
    Io(io::Error),
    Other(String),
}

impl fmt::Display for CliError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            CliError::Flag(ref e) => { e.fmt(f) }
            CliError::Csv(ref e) => { e.fmt(f) }
            CliError::Io(ref e) => { e.fmt(f) }
            CliError::Other(ref s) => { f.write_str(&**s) }
        }
    }
}

impl From<docopt::Error> for CliError {
    fn from(err: docopt::Error) -> CliError {
        CliError::Flag(err)
    }
}

impl From<csv::Error> for CliError {
    fn from(err: csv::Error) -> CliError {
        if !err.is_io_error() {
            return CliError::Csv(err);
        }
        match err.into_kind() {
            csv::ErrorKind::Io(v) => From::from(v),
            _ => unreachable!(),
        }
    }
}

impl From<io::Error> for CliError {
    fn from(err: io::Error) -> CliError {
        CliError::Io(err)
    }
}

impl From<String> for CliError {
    fn from(err: String) -> CliError {
        CliError::Other(err)
    }
}

impl<'a> From<&'a str> for CliError {
    fn from(err: &'a str) -> CliError {
        CliError::Other(err.to_owned())
    }
}

impl From<regex::Error> for CliError {
    fn from(err: regex::Error) -> CliError {
        CliError::Other(format!("{:?}", err))
    }
}
<commit_msg>xsv: add `partition` to command list<commit_after>extern crate byteorder;
extern crate chan;
extern crate csv;
extern crate csv_index;
extern crate docopt;
extern crate filetime;
extern crate num_cpus;
extern crate rand;
extern crate regex;
extern crate serde;
#[macro_use]
extern crate serde_derive;
extern crate stats;
extern crate tabwriter;
extern crate threadpool;

use std::borrow::ToOwned;
use std::env;
use std::fmt;
use std::io;
use std::process;

use docopt::Docopt;

macro_rules! wout {
    ($($arg:tt)*) => ({
        use std::io::Write;
        (writeln!(&mut ::std::io::stdout(), $($arg)*)).unwrap();
    });
}

macro_rules! werr {
    ($($arg:tt)*) => ({
        use std::io::Write;
        (writeln!(&mut ::std::io::stderr(), $($arg)*)).unwrap();
    });
}

macro_rules! fail {
    ($e:expr) => (Err(::std::convert::From::from($e)));
}

macro_rules! command_list {
    () => (
"
    cat         Concatenate by row or column
    count       Count records
    fixlengths  Makes all records have same length
    flatten     Show one field per line
    fmt         Format CSV output (change field delimiter)
    frequency   Show frequency tables
    headers     Show header names
    help        Show this usage message.
    index       Create CSV index for faster access
    input       Read CSV data with special quoting rules
    join        Join CSV files
    partition   Partition CSV data based on a column value
    sample      Randomly sample CSV data
    search      Search CSV data with regexes
    select      Select columns from CSV
    slice       Slice records from CSV
    sort        Sort CSV data
    split       Split CSV data into many files
    stats       Compute basic statistics
    table       Align CSV data into columns
"
    )
}

mod cmd;
mod config;
mod index;
mod select;
mod util;

static USAGE: &'static str = concat!("
Usage:
    xsv <command> [<args>...]
    xsv [options]

Options:
    --list        List all commands available.
    -h, --help    Display this message
    <command> -h  Display the command help message
    --version     Print version info and exit

Commands:", command_list!());

#[derive(Deserialize)]
struct Args {
    arg_command: Option<Command>,
    flag_list: bool,
}

fn main() {
    let args: Args = Docopt::new(USAGE)
                            .and_then(|d| d.options_first(true)
                                           .version(Some(util::version()))
                                           .deserialize())
                            .unwrap_or_else(|e| e.exit());
    if args.flag_list {
        wout!(concat!("Installed commands:", command_list!()));
        return;
    }
    match args.arg_command {
        None => {
            werr!(concat!(
                "xsv is a suite of CSV command line utilities.

Please choose one of the following commands:",
                command_list!()));
            process::exit(0);
        }
        Some(cmd) => {
            match cmd.run() {
                Ok(()) => process::exit(0),
                Err(CliError::Flag(err)) => err.exit(),
                Err(CliError::Csv(err)) => {
                    werr!("{}", err);
                    process::exit(1);
                }
                Err(CliError::Io(ref err))
                        if err.kind() == io::ErrorKind::BrokenPipe => {
                    process::exit(0);
                }
                Err(CliError::Io(err)) => {
                    werr!("{}", err);
                    process::exit(1);
                }
                Err(CliError::Other(msg)) => {
                    werr!("{}", msg);
                    process::exit(1);
                }
            }
        }
    }
}

#[derive(Debug, Deserialize)]
enum Command {
    Cat,
    Count,
    FixLengths,
    Flatten,
    Fmt,
    Frequency,
    Headers,
    Help,
    Index,
    Input,
    Join,
    Partition,
    Sample,
    Search,
    Select,
    Slice,
    Sort,
    Split,
    Stats,
    Table,
}

impl Command {
    fn run(self) -> CliResult<()> {
        let argv: Vec<_> = env::args().map(|v| v.to_owned()).collect();
        let argv: Vec<_> = argv.iter().map(|s| &**s).collect();
        let argv = &*argv;
        match self {
            Command::Cat => cmd::cat::run(argv),
            Command::Count => cmd::count::run(argv),
            Command::FixLengths => cmd::fixlengths::run(argv),
            Command::Flatten => cmd::flatten::run(argv),
            Command::Fmt => cmd::fmt::run(argv),
            Command::Frequency => cmd::frequency::run(argv),
            Command::Headers => cmd::headers::run(argv),
            Command::Help => { wout!("{}", USAGE); Ok(()) }
            Command::Index => cmd::index::run(argv),
            Command::Input => cmd::input::run(argv),
            Command::Join => cmd::join::run(argv),
            Command::Partition => cmd::partition::run(argv),
            Command::Sample => cmd::sample::run(argv),
            Command::Search => cmd::search::run(argv),
            Command::Select => cmd::select::run(argv),
            Command::Slice => cmd::slice::run(argv),
            Command::Sort => cmd::sort::run(argv),
            Command::Split => cmd::split::run(argv),
            Command::Stats => cmd::stats::run(argv),
            Command::Table => cmd::table::run(argv),
        }
    }
}

pub type CliResult<T> = Result<T, CliError>;

#[derive(Debug)]
pub enum CliError {
    Flag(docopt::Error),
    Csv(csv::Error),
    Io(io::Error),
    Other(String),
}

impl fmt::Display for CliError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            CliError::Flag(ref e) => { e.fmt(f) }
            CliError::Csv(ref e) => { e.fmt(f) }
            CliError::Io(ref e) => { e.fmt(f) }
            CliError::Other(ref s) => { f.write_str(&**s) }
        }
    }
}

impl From<docopt::Error> for CliError {
    fn from(err: docopt::Error) -> CliError {
        CliError::Flag(err)
    }
}

impl From<csv::Error> for CliError {
    fn from(err: csv::Error) -> CliError {
        if !err.is_io_error() {
            return CliError::Csv(err);
        }
        match err.into_kind() {
            csv::ErrorKind::Io(v) => From::from(v),
            _ => unreachable!(),
        }
    }
}

impl From<io::Error> for CliError {
    fn from(err: io::Error) -> CliError {
        CliError::Io(err)
    }
}

impl From<String> for CliError {
    fn from(err: String) -> CliError {
        CliError::Other(err)
    }
}

impl<'a> From<&'a str> for CliError {
    fn from(err: &'a str) -> CliError {
        CliError::Other(err.to_owned())
    }
}

impl From<regex::Error> for CliError {
    fn from(err: regex::Error) -> CliError {
        CliError::Other(format!("{:?}", err))
    }
}
<|endoftext|>


<commit_before>use std::io;
use std::fs::File;
use std::io::prelude::*;
extern crate rustc_serialize;
extern crate mustache;
extern crate time;

#[derive(RustcEncodable)]
struct Post {
    categories: String,
    content: String,
    created: String,
    title: String,
}

impl Post {
    fn new() -> Post {
        Post {
            categories: "".to_owned(),
            content: String::new(),
            created: time::strftime("%Y-%m-%d %H:%M:%S +0100", &time::now()).unwrap(),
            title: "".to_owned(),
        }
    }

    fn get_file_name(&self) -> String {
        let lo_title = self.title.to_lowercase().replace(" ", "-");
        let today_date_str = time::strftime("%Y-%m-%d-", &time::now()).unwrap();

        format!("./{}{}.markdown", today_date_str, lo_title)
    }
}

fn print_hint(placeholder: &str) {
    print!("\nPlease input your new post {}: ", placeholder);

    // Force print out, because stdout often gets buffered by line.
    io::stdout().flush().unwrap();
}

fn init_post() -> Post {
    let mut cnt = 1;
    let stdin = io::stdin();
    let mut post = Post::new();

    print_hint("title");

    for line in stdin.lock().lines() {
        //print_hint("HINT!");
        let input: String = line.unwrap().trim().to_owned();

        if input != "" {
            match cnt {
                1 => {
                    post.title = input;
                    print_hint("categories");
                }
                2 => {
                    post.categories = input;
                    //print_hint("... just HIT ENTER to exit.");
                    break
                }
                3 => {
                    break
                }
                _ => panic!("I'm so holy. Gnarf."),
            }
            cnt += 1;
        } else {
            println!("Idiot!");
        }
    }

    post
}

fn handle_template(post: &Post) {
    let mut template_content: String = String::new();
    let template: mustache::Template;
    let mut post_template: File = match File::open("./template.mustache") {
        Err(y) => panic!("{:?}", y),
        Ok(f) => f,
    };

    println!("Found template.");

    if let Err(err) = post_template.read_to_string(&mut template_content) {
        panic!("{:?}", err);
    }

    let mut new_file: File = match File::create(post.get_file_name()) {
        Err(y) => panic!("{:?}", y),
        Ok(f) => f,
    };

    template = mustache::compile_str(&template_content);

    template.render(&mut new_file, post).unwrap();

    println!("Successfully created new blog post: {}", post.get_file_name());
}

fn main() {
    let post: Post = init_post();
    handle_template(&post);
}
<commit_msg>Updated relative directory path to _posts<commit_after>use std::io;
use std::fs::File;
use std::io::prelude::*;
extern crate rustc_serialize;
extern crate mustache;
extern crate time;

#[derive(RustcEncodable)]
struct Post {
    categories: String,
    content: String,
    created: String,
    title: String,
}

impl Post {
    fn new() -> Post {
        Post {
            categories: "".to_owned(),
            content: String::new(),
            created: time::strftime("%Y-%m-%d %H:%M:%S +0100", &time::now()).unwrap(),
            title: "".to_owned(),
        }
    }

    fn get_file_name(&self) -> String {
        let lo_title = self.title.to_lowercase().replace(" ", "-");
        let today_date_str = time::strftime("%Y-%m-%d-", &time::now()).unwrap();

        format!("../_posts/{}{}.markdown", today_date_str, lo_title)
    }
}

fn print_hint(placeholder: &str) {
    print!("\nPlease input your new post {}: ", placeholder);

    // Force print out, because stdout often gets buffered by line.
    io::stdout().flush().unwrap();
}

fn init_post() -> Post {
    let mut cnt = 1;
    let stdin = io::stdin();
    let mut post = Post::new();

    print_hint("title");

    for line in stdin.lock().lines() {
        //print_hint("HINT!");
        let input: String = line.unwrap().trim().to_owned();

        if input != "" {
            match cnt {
                1 => {
                    post.title = input;
                    print_hint("categories");
                }
                2 => {
                    post.categories = input;
                    //print_hint("... just HIT ENTER to exit.");
                    break
                }
                3 => {
                    break
                }
                _ => panic!("I'm so holy. Gnarf."),
            }
            cnt += 1;
        } else {
            println!("Idiot!");
        }
    }

    post
}

fn handle_template(post: &Post) {
    let mut template_content: String = String::new();
    let template: mustache::Template;
    let mut post_template: File = match File::open("./template.mustache") {
        Err(y) => panic!("{:?}", y),
        Ok(f) => f,
    };

    println!("Found template.");

    if let Err(err) = post_template.read_to_string(&mut template_content) {
        panic!("{:?}", err);
    }

    let mut new_file: File = match File::create(post.get_file_name()) {
        Err(y) => panic!("{:?}", y),
        Ok(f) => f,
    };

    template = mustache::compile_str(&template_content);

    template.render(&mut new_file, post).unwrap();

    println!("Successfully created new blog post: {}", post.get_file_name());
}

fn main() {
    let post: Post = init_post();
    handle_template(&post);
}
<|endoftext|>


<commit_before>fn main() {
    println!("Hello, horses!");
}
<commit_msg>basic broken sdl2 bmp image loading<commit_after>extern crate sdl2;

use sdl2::event::{Event};
use sdl2::keyboard::{Keycode};
use sdl2::rect::{Rect};
use sdl2::surface::{Surface};
use std::path::{Path};

fn main() {
    // start sdl2

    let ctx = sdl2::init().unwrap();
    let video_ctx = ctx.video().unwrap();
    let mut timer = ctx.timer().unwrap();

    // Create a window

    let mut window = match video_ctx.window("Gaytracer", 640, 480).position_centered()
                                    .opengl().build() {
        Ok(window) => window,
        Err(err)   => panic!("failed to create window: {}", err)
    };

    // Load image
    let img_surface = Surface::load_bmp(&Path::new("circle.bmp")).unwrap();

    // Create a rendering context
    let mut renderer = match window.renderer().build() {
        Ok(renderer) => renderer,
        Err(err) => panic!("failed to create renderer: {}", err)
    };

    // Set the drawing color to a light blue.
    let _ = renderer.set_draw_color(sdl2::pixels::Color::RGB(101, 208, 246));

    // Clear the buffer, using the light blue color set above.
    let _ = renderer.clear();

    // Set the drawing color to a darker blue.
    let _ = renderer.set_draw_color(sdl2::pixels::Color::RGB(0, 153, 204));

    // Create centered Rect, draw the outline of the Rect in our dark blue color.
    let border_rect = Rect::new(320-64, 240-64, 128, 128).unwrap().unwrap();
    let _ = renderer.draw_rect(border_rect);

    // Create a smaller centered Rect, filling it in the same dark blue.
    let inner_rect = Rect::new(320-60, 240-60, 120, 120).unwrap().unwrap();
    let _ = renderer.fill_rect(inner_rect);

    {
        let mut renderer_surface = renderer.surface_mut().unwrap();
        renderer_surface.blit(None, img_surface, None);
    }

    // Swap our buffer for the present buffer, displaying it.
    let _ = renderer.present();

    let mut events = ctx.event_pump().unwrap();

    // loop until we receive a QuitEvent or escape key pressed
    'event : loop {
        // poll_event returns the most recent event or NoEvent if nothing has happened
        for event in events.poll_iter() {
            match event {
                Event::Quit{..} => break 'event,
                Event::KeyDown{keycode: Option::Some(Keycode::Escape), ..} =>
                    break 'event,
                _ => continue
            }
        }
    }
}
<|endoftext|>


<commit_before>use clap::Clap;
use std::fs::File;

fn normalize_col(col: &str) -> String {
    lazy_static::lazy_static! {
        static ref RE: regex::Regex = regex::Regex::new(r"\(.*?\)").unwrap();
    }
    let mut col = RE
        .replace_all(col, "")
        .to_lowercase()
        .trim()
        .replace(" ", "_")
        .replace(".", "_")
        .replace("-", "_")
        .replace("/", "_")
        .replace("?", "")
        .replace(",", "_")
        .replace("&", "_");
    if !col.chars().next().unwrap().is_alphabetic() {
        col = format!("c_{}", col)
    }
    col
}

fn _create_table(db: &mut rusqlite::Connection, table_name: &str, cols: &[String]) {
    let create_columns = cols
        .iter()
        .map(|c| format!("{} varchar", c))
        .collect::<Vec<String>>()
        .join(", ");
    db.execute(
        &format!("CREATE TABLE {} ({})", table_name, create_columns),
        &[] as &[&dyn rusqlite::types::ToSql],
    )
    .unwrap();
}

fn _load_table_from_path(
    db: &mut rusqlite::Connection,
    table_name: &str,
    path: &str,
    delimiter: u8,
) -> Result<Vec<String>, Box<dyn std::error::Error>> {
    let mut num_rows = 0;
    let f = File::open(path)?;
    let file_size = f.metadata().unwrap().len();
    let mut reader = csv::ReaderBuilder::new()
        .flexible(true)
        .delimiter(delimiter)
        .from_reader(f);

    let normalized_cols =
        reader
            .headers()?
            .iter()
            .map(normalize_col)
            .fold(vec![], |mut v, orig_col| {
                let mut col = orig_col.clone();
                let mut i = 1;
                while v.contains(&col) {
                    col = format!("{}_{}", orig_col, i);
                    i += 1
                }
                v.push(col);
                v
            });
    _create_table(db, table_name, &normalized_cols);

    let insert_query = format!(
        "INSERT INTO {} VALUES ({})",
        table_name,
        normalized_cols
            .iter()
            .map(|_| "?")
            .collect::<Vec<_>>()
            .join(", ")
    );
    let pb = indicatif::ProgressBar::new(file_size);
    pb.set_style(
        indicatif::ProgressStyle::default_bar()
            .template("[{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({eta})")
            .progress_chars("#>-"),
    );
    let mut records = reader.records();
    let tx = db.transaction().unwrap();
    {
        let mut stmt = tx.prepare(&insert_query).expect("tx.prepare() failed");
        while let Some(row) = records.next() {
            let mut row = row?;
            if row.len() < normalized_cols.len() {
                for _ in 0..normalized_cols.len() - row.len() {
                    row.push_field("");
                }
            }
            stmt.execute(&row).unwrap();

            num_rows += 1;
            if num_rows % 10000 == 0 {
                pb.set_position(records.reader().position().byte())
            }
        }
    }
    tx.commit().unwrap();
    pb.finish();

    println!(
        "Loaded {} rows into {}({})",
        num_rows,
        table_name,
        normalized_cols.join(", "),
    );
    Ok(normalized_cols)
}

struct FromAnySqlType {
    value: String,
}

impl rusqlite::types::FromSql for FromAnySqlType {
    fn column_result(
        value: rusqlite::types::ValueRef<'_>,
    ) -> Result<FromAnySqlType, rusqlite::types::FromSqlError> {
        let result = match value {
            rusqlite::types::ValueRef::Null => "null".to_string(),
            rusqlite::types::ValueRef::Integer(v) => v.to_string(),
            rusqlite::types::ValueRef::Real(v) => v.to_string(),
            rusqlite::types::ValueRef::Blob(v) | rusqlite::types::ValueRef::Text(v) => {
                String::from_utf8(v.to_vec()).unwrap()
            }
        };
        Ok(FromAnySqlType { value: result })
    }
}

fn _prepare_query<'a>(
    conn: &'a mut rusqlite::Connection,
    query: &str,
) -> Result<rusqlite::Statement<'a>, String> {
    conn.prepare(&query).map_err(|e| e.to_string())
}

fn _handle_query(conn: &mut rusqlite::Connection, line: &str) -> Result<(), String> {
    let mut stmt = _prepare_query(conn, line)?;

    let mut table = comfy_table::Table::new();
    table.load_preset("││──╞═╪╡┆    ┬┴┌┐└┘");
    table.set_content_arrangement(comfy_table::ContentArrangement::Dynamic);
    let mut title_row = comfy_table::Row::new();
    for col in stmt.column_names() {
        title_row.add_cell(comfy_table::Cell::new(col));
    }
    table.set_header(title_row);

    let mut results = stmt.query(&[] as &[&dyn rusqlite::types::ToSql]).unwrap();
    while let Ok(Some(r)) = results.next() {
        let mut row = comfy_table::Row::new();
        for i in 0..r.column_count() {
            let cell: FromAnySqlType = r.get(i).unwrap();
            row.add_cell(comfy_table::Cell::new(&cell.value));
        }
        table.add_row(row);
    }
    println!("{}", table);
    Ok(())
}

fn _handle_export(conn: &mut rusqlite::Connection, line: &str) -> Result<(), String> {
    lazy_static::lazy_static! {
        static ref RE: regex::Regex = regex::Regex::new(r"^\.export\(([\w_\-\./]+)\) (.*)").unwrap();
    }
    let caps = RE
        .captures(line)
        .ok_or_else(|| "Must match `.export(file-name) SQL`".to_owned())?;
    let destination_path = &caps[1];
    let query = &caps[2];

    let mut stmt = _prepare_query(conn, query)?;

    let mut writer = csv::Writer::from_path(destination_path).unwrap();
    writer.write_record(stmt.column_names()).unwrap();

    let mut results = stmt.query(&[] as &[&dyn rusqlite::types::ToSql]).unwrap();
    while let Ok(Some(r)) = results.next() {
        writer
            .write_record((0..r.column_count()).map(|i| {
                let cell: FromAnySqlType = r.get(i).unwrap();
                cell.value
            }))
            .unwrap();
    }

    Ok(())
}

fn _process_query(conn: &mut rusqlite::Connection, line: &str) {
    let result = if line.starts_with(".export") {
        _handle_export(conn, line)
    } else {
        _handle_query(conn, line)
    };
    if let Err(e) = result {
        println!("{}", e);
    }
}

struct SimpleWordCompleter {
    words: Vec<String>,
}

static BREAK_CHARS: [u8; 4] = [b' ', b'(', b')', b','];
impl SimpleWordCompleter {
    fn new(words: Vec<String>) -> SimpleWordCompleter {
        SimpleWordCompleter { words }
    }
}

impl rustyline::Helper for SimpleWordCompleter {}

impl rustyline::hint::Hinter for SimpleWordCompleter {
    fn hint(&self, _line: &str, _pos: usize, _ctx: &rustyline::Context<'_>) -> Option<String> {
        None
    }
}

impl rustyline::highlight::Highlighter for SimpleWordCompleter {}

impl rustyline::validate::Validator for SimpleWordCompleter {}

impl rustyline::completion::Completer for SimpleWordCompleter {
    type Candidate = String;

    fn complete(
        &self,
        line: &str,
        pos: usize,
        _ctx: &rustyline::Context<'_>,
    ) -> rustyline::Result<(usize, Vec<String>)> {
        let (start, word) = rustyline::completion::extract_word(line, pos, None, &BREAK_CHARS);

        let matches = self
            .words
            .iter()
            .filter(|w| w.starts_with(word))
            .cloned()
            .collect();
        Ok((start, matches))
    }
}

#[derive(Clap)]
struct Opts {
    #[clap(long, about="Use ',' as the delimiter for the CSV")]
    comma: bool,
    #[clap(long, about="Use '|' as the delimiter for the CSV")]
    pipe: bool,
    #[clap(long, about="Use '\\t' as the delimiter for the CSV")]
    tab: bool,

    #[clap()]
    paths: Vec<String>,
}

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let opts = Opts::parse();

    if [opts.comma, opts.pipe, opts.tab]
        .iter()
        .filter(|&&v| v)
        .count()
        > 1
    {
        eprintln!("Can't pass more than one of --comma, --pipe, and --tab");
        std::process::exit(1);
    }

    let delim = if opts.comma {
        b','
    } else if opts.pipe {
        b'|'
    } else if opts.tab {
        b'\t'
    } else {
        b','
    };

    let mut conn = rusqlite::Connection::open_in_memory().unwrap();

    let mut base_words = [
        "distinct", "select", "from", "group", "by", "order", "where", "count", "limit", "offset",
        ".export",
    ]
    .iter()
    .map(|&s| s.to_string())
    .collect::<Vec<String>>();

    if opts.paths.len() == 1 {
        let mut col_names = _load_table_from_path(&mut conn, "t", &opts.paths[0], delim)?;
        base_words.append(&mut col_names);
    } else {
        for (idx, path) in opts.paths.iter().enumerate() {
            let mut col_names =
                _load_table_from_path(&mut conn, &format!("t{}", idx + 1), path, delim)?;
            base_words.append(&mut col_names);
        }
    }

    let completer = SimpleWordCompleter::new(base_words);
    let mut rl = rustyline::Editor::new();
    rl.set_helper(Some(completer));
    loop {
        match rl.readline("> ") {
            Ok(line) => {
                if line.trim().is_empty() {
                    continue;
                }
                _process_query(&mut conn, &line);
                rl.add_history_entry(line);
            }
            Err(rustyline::error::ReadlineError::Interrupted) => {
                println!("Interrupted");
                continue;
            }
            Err(rustyline::error::ReadlineError::Eof) => {
                break;
            }
            Err(err) => {
                println!("Error: {}", err);
                break;
            }
        }
    }

    Ok(())
}

#[cfg(test)]
mod test {
    use super::normalize_col;

    #[test]
    fn test_normalize_col() {
        for (value, expected) in &[("abc", "abc"), ("2/6/2000", "c_2_6_2000")] {
            assert_eq!(&&normalize_col(value), expected);
        }
    }
}
<commit_msg>cargo fmt<commit_after>use clap::Clap;
use std::fs::File;

fn normalize_col(col: &str) -> String {
    lazy_static::lazy_static! {
        static ref RE: regex::Regex = regex::Regex::new(r"\(.*?\)").unwrap();
    }
    let mut col = RE
        .replace_all(col, "")
        .to_lowercase()
        .trim()
        .replace(" ", "_")
        .replace(".", "_")
        .replace("-", "_")
        .replace("/", "_")
        .replace("?", "")
        .replace(",", "_")
        .replace("&", "_");
    if !col.chars().next().unwrap().is_alphabetic() {
        col = format!("c_{}", col)
    }
    col
}

fn _create_table(db: &mut rusqlite::Connection, table_name: &str, cols: &[String]) {
    let create_columns = cols
        .iter()
        .map(|c| format!("{} varchar", c))
        .collect::<Vec<String>>()
        .join(", ");
    db.execute(
        &format!("CREATE TABLE {} ({})", table_name, create_columns),
        &[] as &[&dyn rusqlite::types::ToSql],
    )
    .unwrap();
}

fn _load_table_from_path(
    db: &mut rusqlite::Connection,
    table_name: &str,
    path: &str,
    delimiter: u8,
) -> Result<Vec<String>, Box<dyn std::error::Error>> {
    let mut num_rows = 0;
    let f = File::open(path)?;
    let file_size = f.metadata().unwrap().len();
    let mut reader = csv::ReaderBuilder::new()
        .flexible(true)
        .delimiter(delimiter)
        .from_reader(f);

    let normalized_cols =
        reader
            .headers()?
            .iter()
            .map(normalize_col)
            .fold(vec![], |mut v, orig_col| {
                let mut col = orig_col.clone();
                let mut i = 1;
                while v.contains(&col) {
                    col = format!("{}_{}", orig_col, i);
                    i += 1
                }
                v.push(col);
                v
            });
    _create_table(db, table_name, &normalized_cols);

    let insert_query = format!(
        "INSERT INTO {} VALUES ({})",
        table_name,
        normalized_cols
            .iter()
            .map(|_| "?")
            .collect::<Vec<_>>()
            .join(", ")
    );
    let pb = indicatif::ProgressBar::new(file_size);
    pb.set_style(
        indicatif::ProgressStyle::default_bar()
            .template("[{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({eta})")
            .progress_chars("#>-"),
    );
    let mut records = reader.records();
    let tx = db.transaction().unwrap();
    {
        let mut stmt = tx.prepare(&insert_query).expect("tx.prepare() failed");
        while let Some(row) = records.next() {
            let mut row = row?;
            if row.len() < normalized_cols.len() {
                for _ in 0..normalized_cols.len() - row.len() {
                    row.push_field("");
                }
            }
            stmt.execute(&row).unwrap();

            num_rows += 1;
            if num_rows % 10000 == 0 {
                pb.set_position(records.reader().position().byte())
            }
        }
    }
    tx.commit().unwrap();
    pb.finish();

    println!(
        "Loaded {} rows into {}({})",
        num_rows,
        table_name,
        normalized_cols.join(", "),
    );
    Ok(normalized_cols)
}

struct FromAnySqlType {
    value: String,
}

impl rusqlite::types::FromSql for FromAnySqlType {
    fn column_result(
        value: rusqlite::types::ValueRef<'_>,
    ) -> Result<FromAnySqlType, rusqlite::types::FromSqlError> {
        let result = match value {
            rusqlite::types::ValueRef::Null => "null".to_string(),
            rusqlite::types::ValueRef::Integer(v) => v.to_string(),
            rusqlite::types::ValueRef::Real(v) => v.to_string(),
            rusqlite::types::ValueRef::Blob(v) | rusqlite::types::ValueRef::Text(v) => {
                String::from_utf8(v.to_vec()).unwrap()
            }
        };
        Ok(FromAnySqlType { value: result })
    }
}

fn _prepare_query<'a>(
    conn: &'a mut rusqlite::Connection,
    query: &str,
) -> Result<rusqlite::Statement<'a>, String> {
    conn.prepare(&query).map_err(|e| e.to_string())
}

fn _handle_query(conn: &mut rusqlite::Connection, line: &str) -> Result<(), String> {
    let mut stmt = _prepare_query(conn, line)?;

    let mut table = comfy_table::Table::new();
    table.load_preset("││──╞═╪╡┆    ┬┴┌┐└┘");
    table.set_content_arrangement(comfy_table::ContentArrangement::Dynamic);
    let mut title_row = comfy_table::Row::new();
    for col in stmt.column_names() {
        title_row.add_cell(comfy_table::Cell::new(col));
    }
    table.set_header(title_row);

    let mut results = stmt.query(&[] as &[&dyn rusqlite::types::ToSql]).unwrap();
    while let Ok(Some(r)) = results.next() {
        let mut row = comfy_table::Row::new();
        for i in 0..r.column_count() {
            let cell: FromAnySqlType = r.get(i).unwrap();
            row.add_cell(comfy_table::Cell::new(&cell.value));
        }
        table.add_row(row);
    }
    println!("{}", table);
    Ok(())
}

fn _handle_export(conn: &mut rusqlite::Connection, line: &str) -> Result<(), String> {
    lazy_static::lazy_static! {
        static ref RE: regex::Regex = regex::Regex::new(r"^\.export\(([\w_\-\./]+)\) (.*)").unwrap();
    }
    let caps = RE
        .captures(line)
        .ok_or_else(|| "Must match `.export(file-name) SQL`".to_owned())?;
    let destination_path = &caps[1];
    let query = &caps[2];

    let mut stmt = _prepare_query(conn, query)?;

    let mut writer = csv::Writer::from_path(destination_path).unwrap();
    writer.write_record(stmt.column_names()).unwrap();

    let mut results = stmt.query(&[] as &[&dyn rusqlite::types::ToSql]).unwrap();
    while let Ok(Some(r)) = results.next() {
        writer
            .write_record((0..r.column_count()).map(|i| {
                let cell: FromAnySqlType = r.get(i).unwrap();
                cell.value
            }))
            .unwrap();
    }

    Ok(())
}

fn _process_query(conn: &mut rusqlite::Connection, line: &str) {
    let result = if line.starts_with(".export") {
        _handle_export(conn, line)
    } else {
        _handle_query(conn, line)
    };
    if let Err(e) = result {
        println!("{}", e);
    }
}

struct SimpleWordCompleter {
    words: Vec<String>,
}

static BREAK_CHARS: [u8; 4] = [b' ', b'(', b')', b','];
impl SimpleWordCompleter {
    fn new(words: Vec<String>) -> SimpleWordCompleter {
        SimpleWordCompleter { words }
    }
}

impl rustyline::Helper for SimpleWordCompleter {}

impl rustyline::hint::Hinter for SimpleWordCompleter {
    fn hint(&self, _line: &str, _pos: usize, _ctx: &rustyline::Context<'_>) -> Option<String> {
        None
    }
}

impl rustyline::highlight::Highlighter for SimpleWordCompleter {}

impl rustyline::validate::Validator for SimpleWordCompleter {}

impl rustyline::completion::Completer for SimpleWordCompleter {
    type Candidate = String;

    fn complete(
        &self,
        line: &str,
        pos: usize,
        _ctx: &rustyline::Context<'_>,
    ) -> rustyline::Result<(usize, Vec<String>)> {
        let (start, word) = rustyline::completion::extract_word(line, pos, None, &BREAK_CHARS);

        let matches = self
            .words
            .iter()
            .filter(|w| w.starts_with(word))
            .cloned()
            .collect();
        Ok((start, matches))
    }
}

#[derive(Clap)]
struct Opts {
    #[clap(long, about = "Use ',' as the delimiter for the CSV")]
    comma: bool,
    #[clap(long, about = "Use '|' as the delimiter for the CSV")]
    pipe: bool,
    #[clap(long, about = "Use '\\t' as the delimiter for the CSV")]
    tab: bool,

    #[clap()]
    paths: Vec<String>,
}

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let opts = Opts::parse();

    if [opts.comma, opts.pipe, opts.tab]
        .iter()
        .filter(|&&v| v)
        .count()
        > 1
    {
        eprintln!("Can't pass more than one of --comma, --pipe, and --tab");
        std::process::exit(1);
    }

    let delim = if opts.comma {
        b','
    } else if opts.pipe {
        b'|'
    } else if opts.tab {
        b'\t'
    } else {
        b','
    };

    let mut conn = rusqlite::Connection::open_in_memory().unwrap();

    let mut base_words = [
        "distinct", "select", "from", "group", "by", "order", "where", "count", "limit", "offset",
        ".export",
    ]
    .iter()
    .map(|&s| s.to_string())
    .collect::<Vec<String>>();

    if opts.paths.len() == 1 {
        let mut col_names = _load_table_from_path(&mut conn, "t", &opts.paths[0], delim)?;
        base_words.append(&mut col_names);
    } else {
        for (idx, path) in opts.paths.iter().enumerate() {
            let mut col_names =
                _load_table_from_path(&mut conn, &format!("t{}", idx + 1), path, delim)?;
            base_words.append(&mut col_names);
        }
    }

    let completer = SimpleWordCompleter::new(base_words);
    let mut rl = rustyline::Editor::new();
    rl.set_helper(Some(completer));
    loop {
        match rl.readline("> ") {
            Ok(line) => {
                if line.trim().is_empty() {
                    continue;
                }
                _process_query(&mut conn, &line);
                rl.add_history_entry(line);
            }
            Err(rustyline::error::ReadlineError::Interrupted) => {
                println!("Interrupted");
                continue;
            }
            Err(rustyline::error::ReadlineError::Eof) => {
                break;
            }
            Err(err) => {
                println!("Error: {}", err);
                break;
            }
        }
    }

    Ok(())
}

#[cfg(test)]
mod test {
    use super::normalize_col;

    #[test]
    fn test_normalize_col() {
        for (value, expected) in &[("abc", "abc"), ("2/6/2000", "c_2_6_2000")] {
            assert_eq!(&&normalize_col(value), expected);
        }
    }
}
<|endoftext|>