<commit_before>#[macro_use]
extern crate clap;

use petname::Petnames;

use std::collections::HashSet;
use std::fmt;
use std::fs;
use std::io::{self, Write};
use std::path;
use std::process;
use std::str::FromStr;

use clap::Arg;
use rand::seq::IteratorRandom;

fn main() {
    let matches = app().get_matches();
    match run(matches) {
        Err(Error::Disconnected) => {
            process::exit(0);
        }
        Err(e) => {
            eprintln!("Error: {}", e);
            process::exit(1);
        }
        Ok(()) => {
            process::exit(0);
        }
    }
}

enum Error {
    Io(io::Error),
    FileIo(path::PathBuf, io::Error),
    Cardinality(String),
    Alliteration(String),
    Disconnected,
}

impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match *self {
            Error::Io(ref e) => write!(f, "{}", e),
            Error::FileIo(ref path, ref e) => write!(f, "{}: {}", e, path.display()),
            Error::Cardinality(ref message) => write!(f, "cardinality is zero: {}", message),
            Error::Alliteration(ref message) => write!(f, "cannot alliterate: {}", message),
            Error::Disconnected => write!(f, "caller disconnected / stopped reading"),
        }
    }
}

impl From<io::Error> for Error {
    fn from(error: io::Error) -> Self {
        Error::Io(error)
    }
}

fn app<'a, 'b>() -> clap::App<'a, 'b> {
    clap::App::new("rust-petname")
        .version(crate_version!())
        .author(crate_authors!())
        .about("Generate human readable random names.")
        .after_help(concat!(
            "Based on Dustin Kirkland's petname project ",
            "<https://github.com/dustinkirkland/petname>."
        ))
        .arg(
            Arg::with_name("words")
                .short("w")
                .long("words")
                .value_name("WORDS")
                .default_value("2")
                .help("Number of words in name")
                .takes_value(true)
                .validator(can_be_parsed::<u8>),
        )
        .arg(
            Arg::with_name("separator")
                .short("s")
                .long("separator")
                .value_name("SEP")
                .default_value("-")
                .help("Separator between words")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("complexity")
                .short("c")
                .long("complexity")
                .value_name("COM")
                .possible_values(&["0", "1", "2"])
                .hide_possible_values(true)
                .default_value("0")
                .help("Use small words (0), medium words (1), or large words (2)")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("directory")
                .short("d")
                .long("dir")
                .value_name("DIR")
                .help("Directory containing adjectives.txt, adverbs.txt, names.txt")
                .conflicts_with("complexity")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("count")
                .long("count")
                .value_name("COUNT")
                .default_value("1")
                .help("Generate multiple names; pass 0 to produce infinite names!")
                .takes_value(true)
                .validator(can_be_parsed::<usize>),
        )
        .arg(
            Arg::with_name("letters")
                .short("l")
                .long("letters")
                .value_name("LETTERS")
                .default_value("0")
                .help("Maxiumum number of letters in each word; 0 for unlimited")
                .takes_value(true)
                .validator(can_be_parsed::<usize>),
        )
        .arg(
            Arg::with_name("alliterate")
                .short("a")
                .long("alliterate")
                .help("Generate names where each word begins with the same letter")
                .takes_value(false),
        )
        .arg(
            Arg::with_name("alliterate-with")
                .long("alliterate-with")
                .short("A")
                .value_name("LETTER")
                .help("Generate names where each word begins with the given letter")
                .takes_value(true)
                .validator(can_be_parsed::<char>),
        )
        .arg(
            // For compatibility with upstream.
            Arg::with_name("ubuntu")
                .short("u")
                .long("ubuntu")
                .help("Alias; see --alliterate")
                .takes_value(false),
        )
}

fn run(matches: clap::ArgMatches) -> Result<(), Error> {
    // Unwrapping is safe because these options have defaults.
    let opt_separator = matches.value_of("separator").unwrap();
    let opt_words = matches.value_of("words").unwrap();
    let opt_complexity = matches.value_of("complexity").unwrap();
    let opt_count = matches.value_of("count").unwrap();
    let opt_letters = matches.value_of("letters").unwrap();
    let opt_alliterate = matches.is_present("alliterate")
        || matches.is_present("ubuntu")
        || matches.is_present("alliterate-with");

    // Optional arguments without defaults.
    let opt_directory = matches.value_of("directory");
    let opt_alliterate_char = matches
        .value_of("alliterate-with")
        .and_then(|s| s.parse::<char>().ok());

    // Parse numbers. Validated so unwrapping is okay.
    let opt_words: u8 = opt_words.parse().unwrap();
    let opt_count: usize = opt_count.parse().unwrap();
    let opt_letters: usize = opt_letters.parse().unwrap();

    // Load custom word lists, if specified.
    let words = match opt_directory {
        Some(dirname) => Words::load(dirname)?,
        None => Words::Builtin,
    };

    // Select the appropriate word list.
    let mut petnames = match words {
        Words::Custom(ref adjectives, ref adverbs, ref names) => {
            Petnames::init(&adjectives, &adverbs, &names)
        }
        Words::Builtin => match opt_complexity {
            "0" => Petnames::small(),
            "1" => Petnames::medium(),
            "2" => Petnames::large(),
            _ => Petnames::small(),
        },
    };

    // If requested, limit the number of letters.
    if opt_letters != 0 {
        petnames.retain(|s| s.len() <= opt_letters);
    }

    // Check cardinality.
    if petnames.cardinality(opt_words) == 0 {
        return Err(Error::Cardinality(
            "no petnames to choose from; try relaxing constraints".to_string(),
        ));
    }

    // We're going to need a source of randomness.
    let mut rng = rand::thread_rng();

    // Handle alliteration, either by eliminating a specified
    // character, or using a random one.
    if opt_alliterate {
        // We choose the first letter from the intersection of the
        // first letters of each word list in `petnames`.
        let firsts =
            common_first_letters(&petnames.adjectives, &[&petnames.adverbs, &petnames.names]);
        // if a specific character was requested for alliteration,
        // attempt to use it.
        if let Some(c) = opt_alliterate_char {
            if firsts.contains(&c) {
                petnames.retain(|s| s.starts_with(c));
            } else {
                return Err(Error::Alliteration(
                    "no petnames begin with the choosen alliteration character".to_string(),
                ));
            }
        } else {
            // Otherwise choose the first letter at random; fails if
            // there are no letters.
            match firsts.iter().choose(&mut rng) {
                Some(c) => petnames.retain(|s| s.starts_with(*c)),
                None => {
                    return Err(Error::Alliteration(
                        "word lists have no initial letters in common".to_string(),
                    ))
                }
            };
        }
    }

    // Get an iterator for the names we want to print out.
    let names = petnames.iter(&mut rng, opt_words, opt_separator);

    // Manage stdout.
    let stdout = io::stdout();
    let mut writer = io::BufWriter::new(stdout.lock());

    // Stream if count is 0.
    if opt_count == 0 {
        for name in names {
            writeln!(writer, "{}", name).map_err(suppress_disconnect)?;
        }
    } else {
        for name in names.take(opt_count) {
            writeln!(writer, "{}", name)?;
        }
    }

    Ok(())
}

fn can_be_parsed<INTO>(value: String) -> Result<(), String>
where
    INTO: FromStr,
    <INTO as FromStr>::Err: std::fmt::Display,
{
    match value.parse::<INTO>() {
        Err(e) => Err(format!("{}", e)),
        Ok(_) => Ok(()),
    }
}

fn common_first_letters(init: &[&str], more: &[&[&str]]) -> HashSet<char> {
    let mut firsts = first_letters(init);
    let firsts_other: Vec<HashSet<char>> = more.iter().map(|list| first_letters(list)).collect();
    firsts.retain(|c| firsts_other.iter().all(|fs| fs.contains(c)));
    firsts
}

fn first_letters(names: &[&str]) -> HashSet<char> {
    names.iter().filter_map(|s| s.chars().next()).collect()
}

enum Words {
    Custom(String, String, String),
    Builtin,
}

impl Words {
    // Load word lists from the given directory. This function expects to find three
    // files in that directory: `adjectives.txt`, `adverbs.txt`, and `names.txt`.
    // Each should be valid UTF-8, and contain words separated by whitespace.
    fn load<T: AsRef<path::Path>>(dirname: T) -> Result<Self, Error> {
        let dirname = dirname.as_ref();
        Ok(Self::Custom(
            read_file_to_string(dirname.join("adjectives.txt"))?,
            read_file_to_string(dirname.join("adverbs.txt"))?,
            read_file_to_string(dirname.join("names.txt"))?,
        ))
    }
}

fn read_file_to_string<P: AsRef<path::Path>>(path: P) -> Result<String, Error> {
    fs::read_to_string(&path).map_err(|error| Error::FileIo(path.as_ref().to_path_buf(), error))
}

fn suppress_disconnect(err: io::Error) -> Error {
    match err.kind() {
        io::ErrorKind::BrokenPipe => Error::Disconnected,
        _ => err.into(),
    }
}
<commit_msg>Obey clippy.<commit_after>#[macro_use]
extern crate clap;

use petname::Petnames;

use std::collections::HashSet;
use std::fmt;
use std::fs;
use std::io::{self, Write};
use std::path;
use std::process;
use std::str::FromStr;

use clap::Arg;
use rand::seq::IteratorRandom;

fn main() {
    let matches = app().get_matches();
    match run(matches) {
        Err(Error::Disconnected) => {
            process::exit(0);
        }
        Err(e) => {
            eprintln!("Error: {}", e);
            process::exit(1);
        }
        Ok(()) => {
            process::exit(0);
        }
    }
}

enum Error {
    Io(io::Error),
    FileIo(path::PathBuf, io::Error),
    Cardinality(String),
    Alliteration(String),
    Disconnected,
}

impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match *self {
            Error::Io(ref e) => write!(f, "{}", e),
            Error::FileIo(ref path, ref e) => write!(f, "{}: {}", e, path.display()),
            Error::Cardinality(ref message) => write!(f, "cardinality is zero: {}", message),
            Error::Alliteration(ref message) => write!(f, "cannot alliterate: {}", message),
            Error::Disconnected => write!(f, "caller disconnected / stopped reading"),
        }
    }
}

impl From<io::Error> for Error {
    fn from(error: io::Error) -> Self {
        Error::Io(error)
    }
}

fn app<'a, 'b>() -> clap::App<'a, 'b> {
    clap::App::new("rust-petname")
        .version(crate_version!())
        .author(crate_authors!())
        .about("Generate human readable random names.")
        .after_help(concat!(
            "Based on Dustin Kirkland's petname project ",
            "<https://github.com/dustinkirkland/petname>."
        ))
        .arg(
            Arg::with_name("words")
                .short("w")
                .long("words")
                .value_name("WORDS")
                .default_value("2")
                .help("Number of words in name")
                .takes_value(true)
                .validator(can_be_parsed::<u8>),
        )
        .arg(
            Arg::with_name("separator")
                .short("s")
                .long("separator")
                .value_name("SEP")
                .default_value("-")
                .help("Separator between words")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("complexity")
                .short("c")
                .long("complexity")
                .value_name("COM")
                .possible_values(&["0", "1", "2"])
                .hide_possible_values(true)
                .default_value("0")
                .help("Use small words (0), medium words (1), or large words (2)")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("directory")
                .short("d")
                .long("dir")
                .value_name("DIR")
                .help("Directory containing adjectives.txt, adverbs.txt, names.txt")
                .conflicts_with("complexity")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("count")
                .long("count")
                .value_name("COUNT")
                .default_value("1")
                .help("Generate multiple names; pass 0 to produce infinite names!")
                .takes_value(true)
                .validator(can_be_parsed::<usize>),
        )
        .arg(
            Arg::with_name("letters")
                .short("l")
                .long("letters")
                .value_name("LETTERS")
                .default_value("0")
                .help("Maxiumum number of letters in each word; 0 for unlimited")
                .takes_value(true)
                .validator(can_be_parsed::<usize>),
        )
        .arg(
            Arg::with_name("alliterate")
                .short("a")
                .long("alliterate")
                .help("Generate names where each word begins with the same letter")
                .takes_value(false),
        )
        .arg(
            Arg::with_name("alliterate-with")
                .long("alliterate-with")
                .short("A")
                .value_name("LETTER")
                .help("Generate names where each word begins with the given letter")
                .takes_value(true)
                .validator(can_be_parsed::<char>),
        )
        .arg(
            // For compatibility with upstream.
            Arg::with_name("ubuntu")
                .short("u")
                .long("ubuntu")
                .help("Alias; see --alliterate")
                .takes_value(false),
        )
}

fn run(matches: clap::ArgMatches) -> Result<(), Error> {
    // Unwrapping is safe because these options have defaults.
    let opt_separator = matches.value_of("separator").unwrap();
    let opt_words = matches.value_of("words").unwrap();
    let opt_complexity = matches.value_of("complexity").unwrap();
    let opt_count = matches.value_of("count").unwrap();
    let opt_letters = matches.value_of("letters").unwrap();
    let opt_alliterate = matches.is_present("alliterate")
        || matches.is_present("ubuntu")
        || matches.is_present("alliterate-with");

    // Optional arguments without defaults.
    let opt_directory = matches.value_of("directory");
    let opt_alliterate_char = matches
        .value_of("alliterate-with")
        .and_then(|s| s.parse::<char>().ok());

    // Parse numbers. Validated so unwrapping is okay.
    let opt_words: u8 = opt_words.parse().unwrap();
    let opt_count: usize = opt_count.parse().unwrap();
    let opt_letters: usize = opt_letters.parse().unwrap();

    // Load custom word lists, if specified.
    let words = match opt_directory {
        Some(dirname) => Words::load(dirname)?,
        None => Words::Builtin,
    };

    // Select the appropriate word list.
    let mut petnames = match words {
        Words::Custom(ref adjectives, ref adverbs, ref names) => {
            Petnames::init(adjectives, adverbs, names)
        }
        Words::Builtin => match opt_complexity {
            "0" => Petnames::small(),
            "1" => Petnames::medium(),
            "2" => Petnames::large(),
            _ => Petnames::small(),
        },
    };

    // If requested, limit the number of letters.
    if opt_letters != 0 {
        petnames.retain(|s| s.len() <= opt_letters);
    }

    // Check cardinality.
    if petnames.cardinality(opt_words) == 0 {
        return Err(Error::Cardinality(
            "no petnames to choose from; try relaxing constraints".to_string(),
        ));
    }

    // We're going to need a source of randomness.
    let mut rng = rand::thread_rng();

    // Handle alliteration, either by eliminating a specified
    // character, or using a random one.
    if opt_alliterate {
        // We choose the first letter from the intersection of the
        // first letters of each word list in `petnames`.
        let firsts =
            common_first_letters(&petnames.adjectives, &[&petnames.adverbs, &petnames.names]);
        // if a specific character was requested for alliteration,
        // attempt to use it.
        if let Some(c) = opt_alliterate_char {
            if firsts.contains(&c) {
                petnames.retain(|s| s.starts_with(c));
            } else {
                return Err(Error::Alliteration(
                    "no petnames begin with the choosen alliteration character".to_string(),
                ));
            }
        } else {
            // Otherwise choose the first letter at random; fails if
            // there are no letters.
            match firsts.iter().choose(&mut rng) {
                Some(c) => petnames.retain(|s| s.starts_with(*c)),
                None => {
                    return Err(Error::Alliteration(
                        "word lists have no initial letters in common".to_string(),
                    ))
                }
            };
        }
    }

    // Get an iterator for the names we want to print out.
    let names = petnames.iter(&mut rng, opt_words, opt_separator);

    // Manage stdout.
    let stdout = io::stdout();
    let mut writer = io::BufWriter::new(stdout.lock());

    // Stream if count is 0.
    if opt_count == 0 {
        for name in names {
            writeln!(writer, "{}", name).map_err(suppress_disconnect)?;
        }
    } else {
        for name in names.take(opt_count) {
            writeln!(writer, "{}", name)?;
        }
    }

    Ok(())
}

fn can_be_parsed<INTO>(value: String) -> Result<(), String>
where
    INTO: FromStr,
    <INTO as FromStr>::Err: std::fmt::Display,
{
    match value.parse::<INTO>() {
        Err(e) => Err(format!("{}", e)),
        Ok(_) => Ok(()),
    }
}

fn common_first_letters(init: &[&str], more: &[&[&str]]) -> HashSet<char> {
    let mut firsts = first_letters(init);
    let firsts_other: Vec<HashSet<char>> = more.iter().map(|list| first_letters(list)).collect();
    firsts.retain(|c| firsts_other.iter().all(|fs| fs.contains(c)));
    firsts
}

fn first_letters(names: &[&str]) -> HashSet<char> {
    names.iter().filter_map(|s| s.chars().next()).collect()
}

enum Words {
    Custom(String, String, String),
    Builtin,
}

impl Words {
    // Load word lists from the given directory. This function expects to find three
    // files in that directory: `adjectives.txt`, `adverbs.txt`, and `names.txt`.
    // Each should be valid UTF-8, and contain words separated by whitespace.
    fn load<T: AsRef<path::Path>>(dirname: T) -> Result<Self, Error> {
        let dirname = dirname.as_ref();
        Ok(Self::Custom(
            read_file_to_string(dirname.join("adjectives.txt"))?,
            read_file_to_string(dirname.join("adverbs.txt"))?,
            read_file_to_string(dirname.join("names.txt"))?,
        ))
    }
}

fn read_file_to_string<P: AsRef<path::Path>>(path: P) -> Result<String, Error> {
    fs::read_to_string(&path).map_err(|error| Error::FileIo(path.as_ref().to_path_buf(), error))
}

fn suppress_disconnect(err: io::Error) -> Error {
    match err.kind() {
        io::ErrorKind::BrokenPipe => Error::Disconnected,
        _ => err.into(),
    }
}
<|endoftext|>


<commit_before>#![allow(non_snake_case)]

extern crate merfishtools;
extern crate argparse;

use std::io::{stdout, stderr};
use std::str::FromStr;
use argparse::{ArgumentParser, Store, List, StoreTrue, StoreOption};

#[macro_use]
extern crate log;
extern crate fern;
extern crate bio;
extern crate csv;
#[macro_use]
extern crate itertools;
extern crate num;
extern crate rustc_serialize;
extern crate simple_parallel;
extern crate crossbeam;
extern crate regex;
extern crate rgsl;
extern crate ord_subset;

pub mod model;
pub mod io;
pub mod cli;

#[allow(non_camel_case_types)]
#[derive(Debug)]
enum Command {
    exp,
    diffexp,
    multidiffexp,
    //stats,
    None
}

impl FromStr for Command {
    type Err = ();

    fn from_str(src: &str) -> Result<Command, ()> {
        return match src {
            "exp"     => Ok(Command::exp),
            "diffexp" => Ok(Command::diffexp),
            "multidiffexp" => Ok(Command::multidiffexp),
            //"stats"   => Ok(Command::stats),
            _         => Err(()),
        };
    }
}

fn main() {
    let logger_config = fern::DispatchConfig {
        format: Box::new(|msg: &str, level: &log::LogLevel, _: &log::LogLocation| {
            match level {
                &log::LogLevel::Debug => format!("DEBUG: {}", msg),
                _ => msg.to_owned()
            }
        }),
        output: vec![fern::OutputConfig::stderr()],
        level: log::LogLevelFilter::Debug,
    };

    let mut debug = false;
    let mut subcommand = Command::None;
    let mut args = vec![];
    {
        let mut ap = ArgumentParser::new();
        ap.set_description("MERFISHtools is an analysis toolkit for MERFISH data.");
        ap.refer(&mut debug).add_option(&["--debug"], StoreTrue, "Print debugging information.");
        ap.refer(&mut subcommand).required()
            .add_argument("command", Store, "Command to run (exp or diffexp)");
        ap.refer(&mut args)
            .add_argument("arguments", List, "Arguments for command");
        ap.stop_on_first_argument(true);
        ap.parse_args_or_exit();
    }

    if let Err(e) = fern::init_global_logger(
        logger_config,
        if debug { log::LogLevelFilter::Debug } else { log::LogLevelFilter::Info }
    ) {
        panic!("Failed to initialize global logger: {}", e);
    }

    args.insert(0, format!("subcommand {:?}", subcommand));
    match subcommand {
        Command::exp     => exp(args),
        Command::diffexp => diffexp(args),
        Command::multidiffexp => multidiffexp(args),
        //Command::stats   => stats(args),
        Command::None    => {
            error!("Unknown subcommand.");
            std::process::exit(1);
        }
    }
}

fn exp(args: Vec<String>) {
    let mut N = 16;
    let mut m = 4;
    let mut p0 = 0.04;
    let mut p1 = 0.1;
    let mut dist = 4;
    let mut threads = 1;
    let mut estimate_path = None;
    let mut codebook_path: Option<String> = None;
    let mut cells = ".*".to_owned();
    let mut window_width = 100;

    {
        let mut ap = ArgumentParser::new();
        ap.set_description(
r#"For given MERFISH data, calculate expressions for each feature (e.g. gene) in each cell.
Results are provided as PMF (probability mass function) in columns:

    cell
    feature (e.g. gene, rna)
    expression
    posterior probability

Example: 'merfishtools exp < data.txt > expression.txt'"#
        );

        ap.refer(&mut codebook_path).add_option(&["--codebook", "-c"], StoreOption, r#"
Path to codebook definition consisting of tab separated columns: feature, codeword, expressed.
The last column denotes if a codeword is assigned to e.g. a gene for which expression can be expected.
Unless you have misidentification probes (see Chen et al. Science 2015), you will have only ones in this column.
"#).required();
        ap.refer(&mut estimate_path).add_option(&["--estimate"], StoreOption, r#"
Path to write expected value and standard deviation estimates of expression to.
Output is formatted into columns: cell, feature, expected value, standard deviation
"#);
        ap.refer(&mut N).add_option(&["-N"], Store, "Number of bits in readout, i.e., number of hybridization rounds (default: 16).");
        ap.refer(&mut m).add_option(&["-m"], Store, "Number of 1-bits in readout (default: 4).");
        ap.refer(&mut p0).add_option(&["--p0"], Store, "Prior probability of 0->1 error (default: 0.04).");
        ap.refer(&mut p1).add_option(&["--p1"], Store, "Prior probability of 1->0 error (default: 0.1).");
        ap.refer(&mut dist).add_option(&["--hamming-dist", "--dist"], Store, "Hamming distance between encodings (default: 4).");
        ap.refer(&mut threads)
          .add_option(&["--threads", "-t"], Store, "Number of threads to use.");
        ap.refer(&mut cells).add_option(&["--cells"], Store, "Regular expression for cells to select (default: all).");
        ap.refer(&mut window_width).add_option(&["--pmf-window-width"], Store, "Width of the window to calculate PMF for (default: 100).");
        parse_args_or_exit(&ap, args);
    }
    cli::expression(N, m, p0, p1, dist, &codebook_path.unwrap(), estimate_path, threads, &cells, window_width);
}

fn diffexp(args: Vec<String>) {
    let mut threads = 1;
    let mut group1_path = "".to_owned();
    let mut group2_path = "".to_owned();
    let mut pmf_path: Option<String> = None;
    let mut max_fc = 1.0;

    {
        let mut ap = ArgumentParser::new();
        ap.set_description(
r#"For given expression PMFs, calculate differentially expressed features (e.g. genes) between groups of cells given as separate input data.
Results are provided as columns:

    feature (e.g. gene, rna)
    posterior error probability (PEP) for differential expression
    expected FDR when selecting all features down to the current
    bayes factor (BF) for differential expression
    expected log2 fold change of first vs second group
    standard deviation of log2 fold change
    lower and upper bound of 95% credible interval of log2 fold change

Example: "merfishtools diffexp data1.txt data2.txt > diffexp.txt""#
        );

        ap.refer(&mut pmf_path).add_option(&["--pmf"], StoreOption,
r#"Path to write PMF (probability mass function) of Log2 fold change to.
Output is formatted into columns: feature, foldchange, posterior probability"#);
        ap.refer(&mut threads)
          .add_option(&["--threads", "-t"], Store, "Number of threads to use (default: 1).");
        ap.refer(&mut max_fc)
          .add_option(&["--max-null-log2fc"], Store, "Maximum absolute log2 fold change considered as no differential expression (default: 1.0).");
        ap.refer(&mut group1_path).required()
          .add_argument("group1", Store, "Path to expression PMFs for group of cells.");
        ap.refer(&mut group2_path).required()
          .add_argument("group2", Store, "Path to expression PMFs for group of cells.");
        parse_args_or_exit(&ap, args);
    }
    cli::differential_expression(&group1_path, &group2_path, pmf_path, max_fc, threads);
}

fn multidiffexp(args: Vec<String>) {
    let mut threads = 1;
    let mut group_paths = vec![];
    let mut pmf_path: Option<String> = None;
    let mut max_cv = 1.0;

    {
        let mut ap = ArgumentParser::new();
        ap.set_description(
r#"For given expression PMFs, calculate differentially expressed features (e.g. genes) between groups of cells given as separate input data.
Results are provided as columns:

    feature (e.g. gene, rna)
    posterior error probability (PEP) for differential expression
    expected FDR when selecting all features down to the current
    bayes factor (BF) for differential expression
    expected coefficient of variation (CV)
    standard deviation of CV
    lower and upper bound of 95% credible interval of CV

Example: "merfishtools multidiffexp data1.txt data2.txt data3.txt > diffexp.txt""#
        );

        ap.refer(&mut pmf_path).add_option(&["--pmf"], StoreOption,
r#"Path to write PMF (probability mass function) of Log2 fold change to.
Output is formatted into columns: feature, foldchange, posterior probability"#);
        ap.refer(&mut threads)
          .add_option(&["--threads", "-t"], Store, "Number of threads to use (default: 1).");
        ap.refer(&mut max_cv)
          .add_option(&["--max-null-cv"], Store, "Maximum log2 coefficient of variation considered as no differential expression (default: 1.0).");
        ap.refer(&mut group_paths).required()
          .add_argument("groups", List, "Paths to expression PMFs for groups of cells.");
        parse_args_or_exit(&ap, args);
    }
    if group_paths.len() < 2 {
        panic!("At least 2 expression PMFs definition the groups to compare must be given.")
    }
    cli::multi_differential_expression(&group_paths, pmf_path, max_cv, threads);
}

/*fn stats(args: Vec<String>) {
    let mut N = 16;
    let mut m = 4;
    let mut p0 = 0.04;
    let mut p1 = 0.1;
    let mut dist = 4;

    {
        let mut ap = ArgumentParser::new();
        ap.set_description(
r#"For given MERFISH data, calculate per cell statistics.
Results are provided in columns:

cell
expected total expression

Example: 'merfishtools stats < stats.txt > expression.txt'"#
        );

        ap.refer(&mut N).add_option(&["-N"], Store, "Number of bits in readout, i.e., number of hybridization rounds (default: 16).");
        ap.refer(&mut m).add_option(&["-m"], Store, "Number of 1-bits in readout (default: 4).");
        ap.refer(&mut p0).add_option(&["--p0"], Store, "Prior probability of 0->1 error (default: 0.04).");
        ap.refer(&mut p1).add_option(&["--p1"], Store, "Prior probability of 1->0 error (default: 0.1).");
        ap.refer(&mut dist).add_option(&["--hamming-dist", "--dist"], Store, "Hamming distance between encodings (default: 4).");
        parse_args_or_exit(&ap, args);
    }
    cli::stats(N, m, p0, p1, dist);
}*/

fn parse_args_or_exit(ap: &ArgumentParser, args: Vec<String>) {
    match ap.parse(args, &mut stdout(), &mut stderr()) {
        Ok(()) =>  {}
        Err(x) => {
            std::process::exit(x);
        }
    }
}
<commit_msg>Fixed CLI.<commit_after>#![allow(non_snake_case)]

extern crate merfishtools;
extern crate argparse;

use std::io::{stdout, stderr};
use std::str::FromStr;
use argparse::{ArgumentParser, Store, List, StoreTrue, StoreOption};

#[macro_use]
extern crate log;
extern crate fern;
extern crate bio;
extern crate csv;
#[macro_use]
extern crate itertools;
extern crate num;
extern crate rustc_serialize;
extern crate simple_parallel;
extern crate crossbeam;
extern crate regex;
extern crate rgsl;
extern crate ord_subset;

pub mod model;
pub mod io;
pub mod cli;

#[allow(non_camel_case_types)]
#[derive(Debug)]
enum Command {
    exp,
    diffexp,
    multidiffexp,
    //stats,
    None
}

impl FromStr for Command {
    type Err = ();

    fn from_str(src: &str) -> Result<Command, ()> {
        return match src {
            "exp"     => Ok(Command::exp),
            "diffexp" => Ok(Command::diffexp),
            "multidiffexp" => Ok(Command::multidiffexp),
            //"stats"   => Ok(Command::stats),
            _         => Err(()),
        };
    }
}

fn main() {
    let logger_config = fern::DispatchConfig {
        format: Box::new(|msg: &str, level: &log::LogLevel, _: &log::LogLocation| {
            match level {
                &log::LogLevel::Debug => format!("DEBUG: {}", msg),
                _ => msg.to_owned()
            }
        }),
        output: vec![fern::OutputConfig::stderr()],
        level: log::LogLevelFilter::Debug,
    };

    let mut debug = false;
    let mut subcommand = Command::None;
    let mut args = vec![];
    {
        let mut ap = ArgumentParser::new();
        ap.set_description("MERFISHtools is an analysis toolkit for MERFISH data.");
        ap.refer(&mut debug).add_option(&["--debug"], StoreTrue, "Print debugging information.");
        ap.refer(&mut subcommand).required()
            .add_argument("command", Store, "Command to run (exp or diffexp)");
        ap.refer(&mut args)
            .add_argument("arguments", List, "Arguments for command");
        ap.stop_on_first_argument(true);
        ap.parse_args_or_exit();
    }

    if let Err(e) = fern::init_global_logger(
        logger_config,
        if debug { log::LogLevelFilter::Debug } else { log::LogLevelFilter::Info }
    ) {
        panic!("Failed to initialize global logger: {}", e);
    }

    args.insert(0, format!("subcommand {:?}", subcommand));
    match subcommand {
        Command::exp     => exp(args),
        Command::diffexp => diffexp(args),
        Command::multidiffexp => multidiffexp(args),
        //Command::stats   => stats(args),
        Command::None    => {
            error!("Unknown subcommand.");
            std::process::exit(1);
        }
    }
}

fn exp(args: Vec<String>) {
    let mut N = 16;
    let mut m = 4;
    let mut p0 = 0.04;
    let mut p1 = 0.1;
    let mut dist = 4;
    let mut threads = 1;
    let mut estimate_path = None;
    let mut codebook_path: Option<String> = None;
    let mut cells = ".*".to_owned();
    let mut window_width = 100;

    {
        let mut ap = ArgumentParser::new();
        ap.set_description(
r#"For given MERFISH data, calculate expressions for each feature (e.g. gene) in each cell.
Results are provided as PMF (probability mass function) in columns:

    cell
    feature (e.g. gene, rna)
    expression
    posterior probability

Example: 'merfishtools exp < data.txt > expression.txt'"#
        );

        ap.refer(&mut codebook_path).add_option(&["--codebook", "-c"], StoreOption, r#"
Path to codebook definition consisting of tab separated columns: feature, codeword, expressed.
The last column denotes if a codeword is assigned to e.g. a gene for which expression can be expected.
Unless you have misidentification probes (see Chen et al. Science 2015), you will have only ones in this column.
"#).required();
        ap.refer(&mut estimate_path).add_option(&["--estimate"], StoreOption, r#"
Path to write expected value and standard deviation estimates of expression to.
Output is formatted into columns: cell, feature, expected value, standard deviation
"#);
        ap.refer(&mut N).add_option(&["-N"], Store, "Number of bits in readout, i.e., number of hybridization rounds (default: 16).");
        ap.refer(&mut m).add_option(&["-m"], Store, "Number of 1-bits in readout (default: 4).");
        ap.refer(&mut p0).add_option(&["--p0"], Store, "Prior probability of 0->1 error (default: 0.04).");
        ap.refer(&mut p1).add_option(&["--p1"], Store, "Prior probability of 1->0 error (default: 0.1).");
        ap.refer(&mut dist).add_option(&["--hamming-dist", "--dist"], Store, "Hamming distance between encodings (default: 4).");
        ap.refer(&mut threads)
          .add_option(&["--threads", "-t"], Store, "Number of threads to use.");
        ap.refer(&mut cells).add_option(&["--cells"], Store, "Regular expression for cells to select (default: all).");
        ap.refer(&mut window_width).add_option(&["--pmf-window-width"], Store, "Width of the window to calculate PMF for (default: 100).");
        parse_args_or_exit(&ap, args);
    }
    cli::expression(N, m, p0, p1, dist, &codebook_path.unwrap(), estimate_path, threads, &cells, window_width);
}

fn diffexp(args: Vec<String>) {
    let mut threads = 1;
    let mut group1_path = "".to_owned();
    let mut group2_path = "".to_owned();
    let mut pmf_path: Option<String> = None;
    let mut max_fc = 1.0;

    {
        let mut ap = ArgumentParser::new();
        ap.set_description(
r#"For given expression PMFs, calculate differentially expressed features (e.g. genes) between groups of cells given as separate input data.
Results are provided as columns:

    feature (e.g. gene, rna)
    posterior error probability (PEP) for differential expression
    expected FDR when selecting all features down to the current
    bayes factor (BF) for differential expression
    expected log2 fold change of first vs second group
    standard deviation of log2 fold change
    lower and upper bound of 95% credible interval of log2 fold change

Example: "merfishtools diffexp data1.txt data2.txt > diffexp.txt""#
        );

        ap.refer(&mut pmf_path).add_option(&["--pmf"], StoreOption,
r#"Path to write PMF (probability mass function) of Log2 fold change to.
Output is formatted into columns: feature, foldchange, posterior probability"#);
        ap.refer(&mut threads)
          .add_option(&["--threads", "-t"], Store, "Number of threads to use (default: 1).");
        ap.refer(&mut max_fc)
          .add_option(&["--max-null-log2fc"], Store, "Maximum absolute log2 fold change considered as no differential expression (default: 1.0).");
        ap.refer(&mut group1_path).required()
          .add_argument("group1", Store, "Path to expression PMFs for group of cells.");
        ap.refer(&mut group2_path).required()
          .add_argument("group2", Store, "Path to expression PMFs for group of cells.");
        parse_args_or_exit(&ap, args);
    }
    cli::differential_expression(&group1_path, &group2_path, pmf_path, max_fc, threads);
}

fn multidiffexp(args: Vec<String>) {
    let mut threads = 1;
    let mut group_paths = vec![];
    let mut pmf_path: Option<String> = None;
    let mut max_cv = 1.0;

    {
        let mut ap = ArgumentParser::new();
        ap.set_description(
r#"For given expression PMFs, calculate differentially expressed features (e.g. genes) between groups of cells given as separate input data.
Results are provided as columns:

    feature (e.g. gene, rna)
    posterior error probability (PEP) for differential expression
    expected FDR when selecting all features down to the current
    bayes factor (BF) for differential expression
    expected coefficient of variation (CV)
    standard deviation of CV
    lower and upper bound of 95% credible interval of CV

Example: "merfishtools multidiffexp data1.txt data2.txt data3.txt > diffexp.txt""#
        );

        ap.refer(&mut pmf_path).add_option(&["--pmf"], StoreOption,
r#"Path to write PMF (probability mass function) of Log2 fold change to.
Output is formatted into columns: feature, foldchange, posterior probability"#);
        ap.refer(&mut threads)
          .add_option(&["--threads", "-t"], Store, "Number of threads to use (default: 1).");
        ap.refer(&mut max_cv)
          .add_option(&["--max-null-log2cv"], Store, "Maximum log2 coefficient of variation considered as no differential expression (default: 1.0).");
        ap.refer(&mut group_paths).required()
          .add_argument("groups", List, "Paths to expression PMFs for groups of cells.");
        parse_args_or_exit(&ap, args);
    }
    if group_paths.len() < 2 {
        panic!("At least 2 expression PMFs definition the groups to compare must be given.")
    }
    cli::multi_differential_expression(&group_paths, pmf_path, max_cv, threads);
}

/*fn stats(args: Vec<String>) {
    let mut N = 16;
    let mut m = 4;
    let mut p0 = 0.04;
    let mut p1 = 0.1;
    let mut dist = 4;

    {
        let mut ap = ArgumentParser::new();
        ap.set_description(
r#"For given MERFISH data, calculate per cell statistics.
Results are provided in columns:

cell
expected total expression

Example: 'merfishtools stats < stats.txt > expression.txt'"#
        );

        ap.refer(&mut N).add_option(&["-N"], Store, "Number of bits in readout, i.e., number of hybridization rounds (default: 16).");
        ap.refer(&mut m).add_option(&["-m"], Store, "Number of 1-bits in readout (default: 4).");
        ap.refer(&mut p0).add_option(&["--p0"], Store, "Prior probability of 0->1 error (default: 0.04).");
        ap.refer(&mut p1).add_option(&["--p1"], Store, "Prior probability of 1->0 error (default: 0.1).");
        ap.refer(&mut dist).add_option(&["--hamming-dist", "--dist"], Store, "Hamming distance between encodings (default: 4).");
        parse_args_or_exit(&ap, args);
    }
    cli::stats(N, m, p0, p1, dist);
}*/

fn parse_args_or_exit(ap: &ArgumentParser, args: Vec<String>) {
    match ap.parse(args, &mut stdout(), &mut stderr()) {
        Ok(()) =>  {}
        Err(x) => {
            std::process::exit(x);
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2017 Serde Developers
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! # Serde
//!
//! Serde is a framework for ***ser***ializing and ***de***serializing Rust data
//! structures efficiently and generically.
//!
//! The Serde ecosystem consists of data structures that know how to serialize
//! and deserialize themselves along with data formats that know how to
//! serialize and deserialize other things. Serde provides the layer by which
//! these two groups interact with each other, allowing any supported data
//! structure to be serialized and deserialized using any supported data format.
//!
//! See the Serde website [https://serde.rs/] for additional documentation and
//! usage examples.
//!
//! [https://serde.rs/]: https://serde.rs/
//!
//! ## Design
//!
//! Where many other languages rely on runtime reflection for serializing data,
//! Serde is instead built on Rust's powerful trait system. A data structure
//! that knows how to serialize and deserialize itself is one that implements
//! Serde's `Serialize` and `Deserialize` traits (or uses Serde's derive
//! attribute to automatically generate implementations at compile time). This
//! avoids any overhead of reflection or runtime type information. In fact in
//! many situations the interaction between data structure and data format can
//! be completely optimized away by the Rust compiler, leaving Serde
//! serialization to perform the same speed as a handwritten serializer for the
//! specific selection of data structure and data format.
//!
//! ## Data formats
//!
//! The following is a partial list of data formats that have been implemented
//! for Serde by the community.
//!
//! - [JSON], the ubiquitous JavaScript Object Notation used by many HTTP APIs.
//! - [Bincode], a compact binary format
//!   used for IPC within the Servo rendering engine.
//! - [CBOR], a Concise Binary Object Representation designed for small message
//!   size without the need for version negotiation.
//! - [YAML], a popular human-friendly configuration language that ain't markup
//!   language.
//! - [MessagePack], an efficient binary format that resembles a compact JSON.
//! - [TOML], a minimal configuration format used by [Cargo].
//! - [Pickle], a format common in the Python world.
//! - [Hjson], a variant of JSON designed to be readable and writable by humans.
//! - [BSON], the data storage and network transfer format used by MongoDB.
//! - [URL], the x-www-form-urlencoded format.
//! - [XML], the flexible machine-friendly W3C standard.
//!   *(deserialization only)*
//! - [Envy], a way to deserialize environment variables into Rust structs.
//!   *(deserialization only)*
//! - [Redis], deserialize values from Redis when using [redis-rs].
//!   *(deserialization only)*
//!
//! [JSON]: https://github.com/serde-rs/json
//! [Bincode]: https://github.com/TyOverby/bincode
//! [CBOR]: https://github.com/pyfisch/cbor
//! [YAML]: https://github.com/dtolnay/serde-yaml
//! [MessagePack]: https://github.com/3Hren/msgpack-rust
//! [TOML]: https://github.com/alexcrichton/toml-rs
//! [Pickle]: https://github.com/birkenfeld/serde-pickle
//! [Hjson]: https://github.com/laktak/hjson-rust
//! [BSON]: https://github.com/zonyitoo/bson-rs
//! [URL]: https://github.com/nox/serde_urlencoded
//! [XML]:https://github.com/RReverser/serde-xml-rs
//! [Envy]: https://github.com/softprops/envy
//! [Redis]: https://github.com/OneSignal/serde-redis
//! [Cargo]: http://doc.crates.io/manifest.html
//! [redis-rs]: https://crates.io/crates/redis

////////////////////////////////////////////////////////////////////////////////

// Serde types in rustdoc of other crates get linked to here.
#![doc(html_root_url = "https://docs.rs/serde/0.9.13")]

// Support using Serde without the standard library!
#![cfg_attr(not(feature = "std"), no_std)]

// Unstable functionality only if the user asks for it. For tracking and
// discussion of these features please refer to this issue:
//
//    https://github.com/serde-rs/serde/issues/812
#![cfg_attr(feature = "unstable", feature(nonzero, specialization, zero_one))]
#![cfg_attr(all(feature = "std", feature = "unstable"), feature(into_boxed_c_str))]
#![cfg_attr(feature = "alloc", feature(alloc))]
#![cfg_attr(feature = "collections", feature(collections))]

// Whitelisted clippy lints.
#![cfg_attr(feature = "cargo-clippy", allow(linkedlist, type_complexity, doc_markdown))]

// Blacklisted Rust lints.
#![deny(missing_docs, unused_imports)]

////////////////////////////////////////////////////////////////////////////////

#[cfg(feature = "collections")]
extern crate collections;

#[cfg(feature = "alloc")]
extern crate alloc;

#[cfg(all(feature = "unstable", feature = "std"))]
extern crate core;

/// A facade around all the types we need from the `std`, `core`, `alloc`, and
/// `collections` crates. This avoids elaborate import wrangling having to
/// happen in every module.
mod lib {
    mod core {
        #[cfg(feature = "std")]
        pub use std::*;
        #[cfg(not(feature = "std"))]
        pub use core::*;
    }

    pub use self::core::{cmp, iter, mem, ops, str};
    pub use self::core::{i8, i16, i32, i64, isize};
    pub use self::core::{u8, u16, u32, u64, usize};
    pub use self::core::{f32, f64};

    pub use self::core::clone::{self, Clone};
    pub use self::core::convert::{self, From, Into};
    pub use self::core::default::{self, Default};
    pub use self::core::fmt::{self, Debug, Display};
    pub use self::core::marker::{self, PhantomData};
    pub use self::core::option::{self, Option};
    pub use self::core::result::{self, Result};

    #[cfg(feature = "std")]
    pub use std::borrow::{Cow, ToOwned};
    #[cfg(all(feature = "collections", not(feature = "std")))]
    pub use collections::borrow::{Cow, ToOwned};

    #[cfg(feature = "std")]
    pub use std::string::String;
    #[cfg(all(feature = "collections", not(feature = "std")))]
    pub use collections::string::{String, ToString};

    #[cfg(feature = "std")]
    pub use std::vec::Vec;
    #[cfg(all(feature = "collections", not(feature = "std")))]
    pub use collections::vec::Vec;

    #[cfg(feature = "std")]
    pub use std::boxed::Box;
    #[cfg(all(feature = "alloc", not(feature = "std")))]
    pub use alloc::boxed::Box;

    #[cfg(all(feature = "rc", feature = "std"))]
    pub use std::rc::Rc;
    #[cfg(all(feature = "rc", feature = "alloc", not(feature = "std")))]
    pub use alloc::rc::Rc;

    #[cfg(all(feature = "rc", feature = "std"))]
    pub use std::sync::Arc;
    #[cfg(all(feature = "rc", feature = "alloc", not(feature = "std")))]
    pub use alloc::arc::Arc;

    #[cfg(feature = "std")]
    pub use std::collections::{BinaryHeap, BTreeMap, BTreeSet, LinkedList, VecDeque};
    #[cfg(all(feature = "collections", not(feature = "std")))]
    pub use collections::{BinaryHeap, BTreeMap, BTreeSet, LinkedList, VecDeque};

    #[cfg(feature = "std")]
    pub use std::{error, net};

    #[cfg(feature = "std")]
    pub use std::collections::{HashMap, HashSet};
    #[cfg(feature = "std")]
    pub use std::ffi::{CString, CStr, OsString, OsStr};
    #[cfg(feature = "std")]
    pub use std::hash::{Hash, BuildHasher};
    #[cfg(feature = "std")]
    pub use std::io::Write;
    #[cfg(feature = "std")]
    pub use std::path::{Path, PathBuf};
    #[cfg(feature = "std")]
    pub use std::time::Duration;

    #[cfg(feature = "unstable")]
    pub use core::nonzero::{NonZero, Zeroable};
    #[cfg(feature = "unstable")]
    #[allow(deprecated)] // required for impl Deserialize for NonZero<T>
    pub use core::num::Zero;
}

////////////////////////////////////////////////////////////////////////////////

#[macro_use]
mod macros;

pub mod ser;
pub mod de;

#[doc(inline)]
pub use ser::{Serialize, Serializer};
#[doc(inline)]
pub use de::{Deserialize, Deserializer};

// Generated code uses these to support no_std. Not public API.
#[doc(hidden)]
pub mod export;

// Helpers used by generated code and doc tests. Not public API.
#[doc(hidden)]
pub mod private;

// Re-export #[derive(Serialize, Deserialize)].
//
// This is a workaround for https://github.com/rust-lang/cargo/issues/1286.
// Without this re-export, crates that put Serde derives behind a cfg_attr would
// need to use some silly feature name that depends on both serde and
// serde_derive.
//
//     [features]
//     serde-impls = ["serde", "serde_derive"]
//
//     [dependencies]
//     serde = { version = "1.0", optional = true }
//     serde_derive = { version = "1.0", optional = true }
//
//     # Used like this:
//     # #[cfg(feature = "serde-impls")]
//     # #[macro_use]
//     # extern crate serde_derive;
//     #
//     # #[cfg_attr(feature = "serde-impls", derive(Serialize, Deserialize))]
//     # struct S { /* ... */ }
//
// The re-exported derives allow crates to use "serde" as the name of their
// Serde feature which is more intuitive.
//
//     [dependencies]
//     serde = { version = "1.0", optional = true, features = ["derive"] }
//
//     # Used like this:
//     # #[cfg(feature = "serde")]
//     # #[macro_use]
//     # extern crate serde;
//     #
//     # #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
//     # struct S { /* ... */ }
//
// The reason re-exporting is not enabled by default is that disabling it would
// be annoying for crates that provide handwritten impls or data formats. They
// would need to disable default features and then explicitly re-enable std.
#[cfg(feature = "serde_derive")]
#[allow(unused_imports)]
#[macro_use]
extern crate serde_derive;
#[cfg(feature = "serde_derive")]
#[doc(hidden)]
pub use serde_derive::*;
<commit_msg>Whitelist the zero prefixed literals used in array_impls<commit_after>// Copyright 2017 Serde Developers
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! # Serde
//!
//! Serde is a framework for ***ser***ializing and ***de***serializing Rust data
//! structures efficiently and generically.
//!
//! The Serde ecosystem consists of data structures that know how to serialize
//! and deserialize themselves along with data formats that know how to
//! serialize and deserialize other things. Serde provides the layer by which
//! these two groups interact with each other, allowing any supported data
//! structure to be serialized and deserialized using any supported data format.
//!
//! See the Serde website [https://serde.rs/] for additional documentation and
//! usage examples.
//!
//! [https://serde.rs/]: https://serde.rs/
//!
//! ## Design
//!
//! Where many other languages rely on runtime reflection for serializing data,
//! Serde is instead built on Rust's powerful trait system. A data structure
//! that knows how to serialize and deserialize itself is one that implements
//! Serde's `Serialize` and `Deserialize` traits (or uses Serde's derive
//! attribute to automatically generate implementations at compile time). This
//! avoids any overhead of reflection or runtime type information. In fact in
//! many situations the interaction between data structure and data format can
//! be completely optimized away by the Rust compiler, leaving Serde
//! serialization to perform the same speed as a handwritten serializer for the
//! specific selection of data structure and data format.
//!
//! ## Data formats
//!
//! The following is a partial list of data formats that have been implemented
//! for Serde by the community.
//!
//! - [JSON], the ubiquitous JavaScript Object Notation used by many HTTP APIs.
//! - [Bincode], a compact binary format
//!   used for IPC within the Servo rendering engine.
//! - [CBOR], a Concise Binary Object Representation designed for small message
//!   size without the need for version negotiation.
//! - [YAML], a popular human-friendly configuration language that ain't markup
//!   language.
//! - [MessagePack], an efficient binary format that resembles a compact JSON.
//! - [TOML], a minimal configuration format used by [Cargo].
//! - [Pickle], a format common in the Python world.
//! - [Hjson], a variant of JSON designed to be readable and writable by humans.
//! - [BSON], the data storage and network transfer format used by MongoDB.
//! - [URL], the x-www-form-urlencoded format.
//! - [XML], the flexible machine-friendly W3C standard.
//!   *(deserialization only)*
//! - [Envy], a way to deserialize environment variables into Rust structs.
//!   *(deserialization only)*
//! - [Redis], deserialize values from Redis when using [redis-rs].
//!   *(deserialization only)*
//!
//! [JSON]: https://github.com/serde-rs/json
//! [Bincode]: https://github.com/TyOverby/bincode
//! [CBOR]: https://github.com/pyfisch/cbor
//! [YAML]: https://github.com/dtolnay/serde-yaml
//! [MessagePack]: https://github.com/3Hren/msgpack-rust
//! [TOML]: https://github.com/alexcrichton/toml-rs
//! [Pickle]: https://github.com/birkenfeld/serde-pickle
//! [Hjson]: https://github.com/laktak/hjson-rust
//! [BSON]: https://github.com/zonyitoo/bson-rs
//! [URL]: https://github.com/nox/serde_urlencoded
//! [XML]:https://github.com/RReverser/serde-xml-rs
//! [Envy]: https://github.com/softprops/envy
//! [Redis]: https://github.com/OneSignal/serde-redis
//! [Cargo]: http://doc.crates.io/manifest.html
//! [redis-rs]: https://crates.io/crates/redis

////////////////////////////////////////////////////////////////////////////////

// Serde types in rustdoc of other crates get linked to here.
#![doc(html_root_url = "https://docs.rs/serde/0.9.13")]

// Support using Serde without the standard library!
#![cfg_attr(not(feature = "std"), no_std)]

// Unstable functionality only if the user asks for it. For tracking and
// discussion of these features please refer to this issue:
//
//    https://github.com/serde-rs/serde/issues/812
#![cfg_attr(feature = "unstable", feature(nonzero, specialization, zero_one))]
#![cfg_attr(all(feature = "std", feature = "unstable"), feature(into_boxed_c_str))]
#![cfg_attr(feature = "alloc", feature(alloc))]
#![cfg_attr(feature = "collections", feature(collections))]

// Whitelisted clippy lints.
#![cfg_attr(feature = "cargo-clippy", allow(doc_markdown))]
#![cfg_attr(feature = "cargo-clippy", allow(linkedlist))]
#![cfg_attr(feature = "cargo-clippy", allow(type_complexity))]
#![cfg_attr(feature = "cargo-clippy", allow(zero_prefixed_literal))]

// Blacklisted Rust lints.
#![deny(missing_docs, unused_imports)]

////////////////////////////////////////////////////////////////////////////////

#[cfg(feature = "collections")]
extern crate collections;

#[cfg(feature = "alloc")]
extern crate alloc;

#[cfg(all(feature = "unstable", feature = "std"))]
extern crate core;

/// A facade around all the types we need from the `std`, `core`, `alloc`, and
/// `collections` crates. This avoids elaborate import wrangling having to
/// happen in every module.
mod lib {
    mod core {
        #[cfg(feature = "std")]
        pub use std::*;
        #[cfg(not(feature = "std"))]
        pub use core::*;
    }

    pub use self::core::{cmp, iter, mem, ops, str};
    pub use self::core::{i8, i16, i32, i64, isize};
    pub use self::core::{u8, u16, u32, u64, usize};
    pub use self::core::{f32, f64};

    pub use self::core::clone::{self, Clone};
    pub use self::core::convert::{self, From, Into};
    pub use self::core::default::{self, Default};
    pub use self::core::fmt::{self, Debug, Display};
    pub use self::core::marker::{self, PhantomData};
    pub use self::core::option::{self, Option};
    pub use self::core::result::{self, Result};

    #[cfg(feature = "std")]
    pub use std::borrow::{Cow, ToOwned};
    #[cfg(all(feature = "collections", not(feature = "std")))]
    pub use collections::borrow::{Cow, ToOwned};

    #[cfg(feature = "std")]
    pub use std::string::String;
    #[cfg(all(feature = "collections", not(feature = "std")))]
    pub use collections::string::{String, ToString};

    #[cfg(feature = "std")]
    pub use std::vec::Vec;
    #[cfg(all(feature = "collections", not(feature = "std")))]
    pub use collections::vec::Vec;

    #[cfg(feature = "std")]
    pub use std::boxed::Box;
    #[cfg(all(feature = "alloc", not(feature = "std")))]
    pub use alloc::boxed::Box;

    #[cfg(all(feature = "rc", feature = "std"))]
    pub use std::rc::Rc;
    #[cfg(all(feature = "rc", feature = "alloc", not(feature = "std")))]
    pub use alloc::rc::Rc;

    #[cfg(all(feature = "rc", feature = "std"))]
    pub use std::sync::Arc;
    #[cfg(all(feature = "rc", feature = "alloc", not(feature = "std")))]
    pub use alloc::arc::Arc;

    #[cfg(feature = "std")]
    pub use std::collections::{BinaryHeap, BTreeMap, BTreeSet, LinkedList, VecDeque};
    #[cfg(all(feature = "collections", not(feature = "std")))]
    pub use collections::{BinaryHeap, BTreeMap, BTreeSet, LinkedList, VecDeque};

    #[cfg(feature = "std")]
    pub use std::{error, net};

    #[cfg(feature = "std")]
    pub use std::collections::{HashMap, HashSet};
    #[cfg(feature = "std")]
    pub use std::ffi::{CString, CStr, OsString, OsStr};
    #[cfg(feature = "std")]
    pub use std::hash::{Hash, BuildHasher};
    #[cfg(feature = "std")]
    pub use std::io::Write;
    #[cfg(feature = "std")]
    pub use std::path::{Path, PathBuf};
    #[cfg(feature = "std")]
    pub use std::time::Duration;

    #[cfg(feature = "unstable")]
    pub use core::nonzero::{NonZero, Zeroable};
    #[cfg(feature = "unstable")]
    #[allow(deprecated)] // required for impl Deserialize for NonZero<T>
    pub use core::num::Zero;
}

////////////////////////////////////////////////////////////////////////////////

#[macro_use]
mod macros;

pub mod ser;
pub mod de;

#[doc(inline)]
pub use ser::{Serialize, Serializer};
#[doc(inline)]
pub use de::{Deserialize, Deserializer};

// Generated code uses these to support no_std. Not public API.
#[doc(hidden)]
pub mod export;

// Helpers used by generated code and doc tests. Not public API.
#[doc(hidden)]
pub mod private;

// Re-export #[derive(Serialize, Deserialize)].
//
// This is a workaround for https://github.com/rust-lang/cargo/issues/1286.
// Without this re-export, crates that put Serde derives behind a cfg_attr would
// need to use some silly feature name that depends on both serde and
// serde_derive.
//
//     [features]
//     serde-impls = ["serde", "serde_derive"]
//
//     [dependencies]
//     serde = { version = "1.0", optional = true }
//     serde_derive = { version = "1.0", optional = true }
//
//     # Used like this:
//     # #[cfg(feature = "serde-impls")]
//     # #[macro_use]
//     # extern crate serde_derive;
//     #
//     # #[cfg_attr(feature = "serde-impls", derive(Serialize, Deserialize))]
//     # struct S { /* ... */ }
//
// The re-exported derives allow crates to use "serde" as the name of their
// Serde feature which is more intuitive.
//
//     [dependencies]
//     serde = { version = "1.0", optional = true, features = ["derive"] }
//
//     # Used like this:
//     # #[cfg(feature = "serde")]
//     # #[macro_use]
//     # extern crate serde;
//     #
//     # #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
//     # struct S { /* ... */ }
//
// The reason re-exporting is not enabled by default is that disabling it would
// be annoying for crates that provide handwritten impls or data formats. They
// would need to disable default features and then explicitly re-enable std.
#[cfg(feature = "serde_derive")]
#[allow(unused_imports)]
#[macro_use]
extern crate serde_derive;
#[cfg(feature = "serde_derive")]
#[doc(hidden)]
pub use serde_derive::*;
<|endoftext|>


<commit_before>#![feature(globs)]
extern crate xlib;
extern crate libc;

use xlib::{XOpenDisplay, XDisplayName, XQueryExtension, XInitExtension};
use libc::{c_int};
use std::ptr;
use xtst::{XRecordCreateContext, XRecordAllClients, XRecordAllocRange, XRecordRange};
use std::mem;
mod xtst;

fn main() {
	let mut a:  i8 = 0;
	let a_ptr: *mut i8 = &mut a;

	let display = unsafe { XOpenDisplay(a_ptr)};
	if display.is_null() {
		fail!("XOpenDisplay() failed!");
	}

	let display_name = unsafe {XDisplayName(a_ptr)};
	let ext_name = "RECORD";

	let arg2:*mut c_int = &mut 1;
	let arg3:*mut c_int = &mut 1;
	let arg4:*mut c_int = &mut 1;
	let has_record = unsafe{XQueryExtension(display, ext_name.to_c_str().as_ptr() as *mut i8,arg2,arg3,arg4)};
	let extension = unsafe{XInitExtension(display, ext_name.to_c_str().as_ptr() as *mut i8)};
	if extension.is_null() {
		fail!("XInitExtension() failde!");
	}

	unsafe {

		let mut recordRange: *mut XRecordRange = XRecordAllocRange();
		(&*recordRange).device_events.first = 2; // KeyPress
		(&*recordRange).device_events.last = 6; // MotionNotify
		let context = XRecordCreateContext(
			display,
			0,
			&mut XRecordAllClients,
			0,
			&mut recordRange,
			0
		);
	}

	println!("display: {}\ndisplay name: {}\nRECORD extension:{}", display, display_name, has_record);
}<commit_msg>success XRecordRange structure allocation<commit_after>#![feature(globs)]
extern crate xlib;
extern crate libc;

use xlib::{XOpenDisplay, XDisplayName, XQueryExtension, XInitExtension};
use libc::{c_int};
use std::ptr;
use xtst::{XRecordCreateContext, XRecordAllClients, XRecordAllocRange, XRecordRange};
use std::mem;
mod xtst;

fn main() {
	let mut a:  i8 = 0;
	let a_ptr: *mut i8 = &mut a;

	let display = unsafe { XOpenDisplay(a_ptr)};
	if display.is_null() {
		fail!("XOpenDisplay() failed!");
	}

	let display_name = unsafe {XDisplayName(a_ptr)};
	let ext_name = "RECORD";

	let arg2:*mut c_int = &mut 1;
	let arg3:*mut c_int = &mut 1;
	let arg4:*mut c_int = &mut 1;
	let has_record = unsafe{XQueryExtension(display, ext_name.to_c_str().as_ptr() as *mut i8,arg2,arg3,arg4)};
	let extension = unsafe{XInitExtension(display, ext_name.to_c_str().as_ptr() as *mut i8)};
	if extension.is_null() {
		fail!("XInitExtension() failde!");
	}

	unsafe {

		let mut recordRange: XRecordRange = *XRecordAllocRange();
		let mut recordRangePtr: *mut *mut XRecordRange = std::mem::transmute(&mut &mut recordRange);
		recordRange.device_events.first = 2; // KeyPress
		recordRange.device_events.last = 6; // MotionNotify
		let context = XRecordCreateContext(
			display,
			0,
			&mut XRecordAllClients,
			0,
			recordRangePtr,
			0
		);
	}

	println!("display: {}\ndisplay name: {}\nRECORD extension:{}", display, display_name, has_record);
}<|endoftext|>


<commit_before>extern crate time;

fn main() {
    let dow = ["Duinday", "Sitaday", "Wikiday", "Tuxday", "Gnuday", "Commonday"];
    let moy = ["Jabim", "Zodrak", "Trogool", "Yanar", "Shkumbe", "Habniah", "Skarl", "Mikon", "Pertunda", "Kib", "Broket"];
    let dpm = 33;
    let dpw = dow.len();
    let mpy = moy.len();
    let (year, month, day, yoff) = current_day(time::now());
    let wd = (yoff + month * 3 + day) % dpw;
    let mut moff = yoff;

    for mm in 0..mpy {
        let spaces = (17 - moy[mm].len()) / 2;
        for _ in 0..spaces {
            print!(" ");
        }
        println!("{}", moy[mm]);

        for dd in 0..dpw {
            let mut abbrd = String::from(dow[dd]);
            abbrd.truncate(2);
            print!("{} ", abbrd);
        }

        let dd = if mm == month { day } else { 1000 };
        moff = print_days(moff, dpm, dpw, dd);
        if mm == 2 {
            println!("--  Peer  Day  --");
        }
        else if mm == 6 {
            println!("--Torrent Feast--");
        }
        else if is_leap_year(year) && mm == 10 {
            println!("--Immersion Day--");
        }
    }

    let mut datestr = format!("{}, {:02} of {}", dow[wd], day, moy[month]);
    let mut daystr = format!("{:02}.{:02}", month + 1, day);
    if day == 34 {
        if month == 2 {
            datestr = format!("Peer Day");
            daystr = format!("P");
        }
        else if month == 6 {
            datestr = format!("Torrent Feast");
            daystr = format!("T");
        }
        else if month == 10 {
            datestr = format!("Immersion Feast");
            daystr = format!("T");
        }
    }
    let hol = format!("{}", what_holiday(month, day));
    println!("Today is {} {}{}.  [{}.{}]", datestr, year, hol, year, daystr);
}

fn print_days(moff: usize, dpm: usize, dpw: usize, highlight: usize) -> usize {
    let mut month = format!("");

    if moff != 0 {
        month = format!("{}\n", month);
    }

    for _ in 0 .. moff {
        month = format!("{}   ", month);
    }

    for day in 1 .. dpm + 1 {
        if (day + moff) % dpw == 1 {
            month = format!("{}\n", month);
        }

        let ch = if day == highlight { ">" } else if day + 1 == highlight { "<" } else { " " };
        month = format!("{}{:2}{}", month, day, ch);
    }

    month = format!("{}\n", month);
    print!("{}", month);
    return if moff == 0 { 3 } else { 0 };
}

fn current_day(current_time: time::Tm) -> (usize, usize, usize, usize) {
    let epoch_days_offset = 1483889;
    let seconds_per_day = 86400;
    let now = current_time.to_timespec();
    let days_since_epoch = (epoch_days_offset + now.sec / seconds_per_day) as i32;
    let days_per_year = 365;
    let leap_year = 4;
    let not_leap_year = 33;
    let days_per_month = 33;

    let mut count = 1;
    let mut days_remaining = days_since_epoch;
    let mut year = 0;
    while days_remaining > days_per_year + 1 {
        days_remaining -= days_per_year;
        year += 1;
        if count > not_leap_year {
            count = 1;
        }
        if count % leap_year == 0 {
            days_remaining -= 1;
        }
        count += 1;
    }
    if days_remaining == days_per_year + 1 && count % leap_year == 0 {
        days_remaining = 0;
        year += 1;
    }

    let mut month = 0;
    while days_remaining > days_per_month {
        month += 1;
        days_remaining -= days_per_month;
        if month == 3 || month == 7 || month == 11 {
            days_remaining -= 1;
        }
    }

    let day = days_remaining;
    return (year, month as usize, day as usize + 1, (year % 2 * 3) as usize);
}

fn is_leap_year(year: usize) -> bool {
    let rem = year % 33;
    return rem != 0 && rem % 4 == 0;
}

fn what_holiday(month: usize, day: usize) -> String {
    let mut result = format!("");
    if month == 0 && day == 1 {
        result = format!(", New Year's Day");
    }
    else if month == 2 && day == 12 {
        result = format!(", Purification Day");
    }
    else if month == 2 && day == 24 {
        result = format!(", Document Freedom Day");
    }
    else if month == 3 && day == 33 {
        result = format!(", Hardware Freedom Day");
    }
    else if month == 5 && day == 4 {
        result = format!(", Harvest Gift");
    }
    else if month == 5 && day == 9 {
        result = format!(", Culture Freedom Day");
    }
    else if month == 7 && day == 29 {
        result = format!(", Familytide");
    }
    else if month == 8 && day == 33 {
        result = format!(", Software Freedom Day");
    }
    else if month == 9 && day == 30 {
        result = format!(", Open Access Day");
    }
    else if month == 10 && day == 21 {
        result = format!(", Freedom Day");
    }
    else if month == 0 && day == 27 {
        result = format!(", International Workers' Day");
    }
    return result;
}

<commit_msg>Combine all month-printing I/O<commit_after>extern crate time;

fn main() {
    let dow = ["Duinday", "Sitaday", "Wikiday", "Tuxday", "Gnuday", "Commonday"];
    let moy = ["Jabim", "Zodrak", "Trogool", "Yanar", "Shkumbe", "Habniah", "Skarl", "Mikon", "Pertunda", "Kib", "Broket"];
    let dpm = 33;
    let dpw = dow.len();
    let mpy = moy.len();
    let (year, month, day, yoff) = current_day(time::now());
    let wd = (yoff + month * 3 + day) % dpw;
    let mut moff = yoff;

    for mm in 0..mpy {
        let spaces = (17 - moy[mm].len()) / 2;
        for _ in 0..spaces {
            print!(" ");
        }
        let mut month_text = format!("{}\n", moy[mm]);

        for dd in 0..dpw {
            let mut abbrd = String::from(dow[dd]);
            abbrd.truncate(2);
            month_text = format!("{}{} ", month_text, abbrd);
        }

        let dd = if mm == month { day } else { 1000 };
        let days = print_days(moff, dpm, dpw, dd);
        month_text = format!("{}{}", month_text, days.0);
        moff = days.1;
        if mm == 2 {
            month_text = format!("{}--  Peer  Day  --", month_text);
        }
        else if mm == 6 {
            month_text = format!("{}--Torrent Feast--", month_text);
        }
        else if is_leap_year(year) && mm == 10 {
            month_text = format!("{}--Immersion Day--", month_text);
        }

        month_text = format!("{}\n", month_text);
        print!("{}", month_text);
    }

    let mut datestr = format!("{}, {:02} of {}", dow[wd], day, moy[month]);
    let mut daystr = format!("{:02}.{:02}", month + 1, day);
    if day == 34 {
        if month == 2 {
            datestr = format!("Peer Day");
            daystr = format!("P");
        }
        else if month == 6 {
            datestr = format!("Torrent Feast");
            daystr = format!("T");
        }
        else if month == 10 {
            datestr = format!("Immersion Feast");
            daystr = format!("T");
        }
    }
    let hol = format!("{}", what_holiday(month, day));
    println!("Today is {} {}{}.  [{}.{}]", datestr, year, hol, year, daystr);
}

fn print_days(moff: usize, dpm: usize, dpw: usize, highlight: usize) -> (String, usize) {
    let mut month = format!("");

    if moff != 0 {
        month = format!("{}\n", month);
    }

    for _ in 0 .. moff {
        month = format!("{}   ", month);
    }

    for day in 1 .. dpm + 1 {
        if (day + moff) % dpw == 1 {
            month = format!("{}\n", month);
        }

        let ch = if day == highlight { ">" } else if day + 1 == highlight { "<" } else { " " };
        month = format!("{}{:2}{}", month, day, ch);
    }

    month = format!("{}\n", month);
    return (month, if moff == 0 { 3 } else { 0 });
}

fn current_day(current_time: time::Tm) -> (usize, usize, usize, usize) {
    let epoch_days_offset = 1483889;
    let seconds_per_day = 86400;
    let now = current_time.to_timespec();
    let days_since_epoch = (epoch_days_offset + now.sec / seconds_per_day) as i32;
    let days_per_year = 365;
    let leap_year = 4;
    let not_leap_year = 33;
    let days_per_month = 33;

    let mut count = 1;
    let mut days_remaining = days_since_epoch;
    let mut year = 0;
    while days_remaining > days_per_year + 1 {
        days_remaining -= days_per_year;
        year += 1;
        if count > not_leap_year {
            count = 1;
        }
        if count % leap_year == 0 {
            days_remaining -= 1;
        }
        count += 1;
    }
    if days_remaining == days_per_year + 1 && count % leap_year == 0 {
        days_remaining = 0;
        year += 1;
    }

    let mut month = 0;
    while days_remaining > days_per_month {
        month += 1;
        days_remaining -= days_per_month;
        if month == 3 || month == 7 || month == 11 {
            days_remaining -= 1;
        }
    }

    let day = days_remaining;
    return (year, month as usize, day as usize + 1, (year % 2 * 3) as usize);
}

fn is_leap_year(year: usize) -> bool {
    let rem = year % 33;
    return rem != 0 && rem % 4 == 0;
}

fn what_holiday(month: usize, day: usize) -> String {
    let mut result = format!("");
    if month == 0 && day == 1 {
        result = format!(", New Year's Day");
    }
    else if month == 2 && day == 12 {
        result = format!(", Purification Day");
    }
    else if month == 2 && day == 24 {
        result = format!(", Document Freedom Day");
    }
    else if month == 3 && day == 33 {
        result = format!(", Hardware Freedom Day");
    }
    else if month == 5 && day == 4 {
        result = format!(", Harvest Gift");
    }
    else if month == 5 && day == 9 {
        result = format!(", Culture Freedom Day");
    }
    else if month == 7 && day == 29 {
        result = format!(", Familytide");
    }
    else if month == 8 && day == 33 {
        result = format!(", Software Freedom Day");
    }
    else if month == 9 && day == 30 {
        result = format!(", Open Access Day");
    }
    else if month == 10 && day == 21 {
        result = format!(", Freedom Day");
    }
    else if month == 0 && day == 27 {
        result = format!(", International Workers' Day");
    }
    return result;
}

<|endoftext|>


<commit_before>extern crate rustc_serialize;
#[macro_use]
extern crate aws_abacus;
#[macro_use]
extern crate log;
extern crate walkdir;
extern crate env_logger;
extern crate docopt;
use docopt::Docopt;
use std::path;
use aws_abacus::elb_log_files;
extern crate chrono;
use chrono::{DateTime, UTC};
use aws_abacus::elb_log_files::ParsingResult;
use std::collections::HashMap;
extern crate urlparse;
use urlparse::{Url, urlparse};

fn main() {
    env_logger::init().unwrap();
    let args: Args = Docopt::new(USAGE)
                            .and_then(|d| d.decode())
                            .unwrap_or_else(|e| e.exit());

    let log_location = &path::Path::new(&args.arg_log_location);
    debug!("Running summary on {}.", log_location.to_str().unwrap());

    let start: Option<DateTime<UTC>> = if args.flag_benchmark {
        Some(UTC::now())
    } else {
        None
    };

    let mut number_of_files = 0;
    let mut number_of_records = 0;
    let mut filenames = Vec::new();
    let mut agg: HashMap<AggregateELBRecord, i64> = HashMap::new();
    match elb_log_files::file_list(log_location, &mut filenames) {
        Ok(num_files) => {
            number_of_files = num_files;
            debug!("Found {} files.", number_of_files);
            number_of_records = elb_log_files::process_files(&filenames, &mut |parsing_result: ParsingResult| {
                parsing_result_handler(parsing_result, &mut agg);
            });
            debug!("Processed {} records in {} files.", number_of_records, num_files);
        },
        Err(e) => {
            println!("ERROR: {}", e);
        },
    };

    for (aggregate, total) in &agg {
        println!(
            "{},{},{},{}",
            aggregate.system_name,
            aggregate.day,
            aggregate.client_address,
            total
        );
    };

    match start {
        Some(s) => {
            let end = UTC::now();
            let time = end - s;
            println!("Processed {} files having {} records in {} milliseconds and produced {} aggregates.",
                number_of_files,
                number_of_records,
                time.num_milliseconds(),
                agg.len()
            );
        },
        None => {},
    };
}

#[derive(Clone, Debug, Eq, Hash, PartialEq)]
struct AggregateELBRecord {
    day: String,
    client_address: String,
    system_name: String,
}

fn parsing_result_handler(parsing_result: ParsingResult, aggregation: &mut HashMap<AggregateELBRecord, i64>) -> () {
    match parsing_result {
        Ok(elb_record) => {
            let url = urlparse(&elb_record.request_url);
            let aer = AggregateELBRecord {
                day: elb_record.timestamp.format("%Y-%m-%d").to_string(),
                client_address: elb_record.client_address.ip().to_string(),
                system_name: parse_system_name(&url).unwrap_or("UNDEFINED_SYSTEM".to_owned()),
            };
            aggregate_record(aer, aggregation);
        },

        Err(errors) => {

        }
    }
}

fn parse_system_name(url: &Url) -> Option<String> {
    match url.get_parsed_query() {
        Some(query_map) => {
            match query_map.get("system") {
                Some(systems) => {
                    Some(systems[0].clone())
                },

                None => {
                    None
                }
            }
        },

        None => {
            None
        }
    }
}

fn aggregate_record(aggregate_record: AggregateELBRecord, aggregation: &mut HashMap<AggregateELBRecord, i64>) -> () {
    let total = aggregation.entry(aggregate_record).or_insert(0);
    *total += 1;
}

const USAGE: &'static str = "
aws-abacus

Usage:
  aws-abacus <log-location>
  aws-abacus (-d | --debug | -b | --benchmark) <log-location>
  aws-abacus (-h | --help)

Options:
  -h --help     Show this screen.
  -d --debug    Turn on debug output
";

#[derive(Debug, RustcDecodable)]
struct Args {
    arg_log_location: String,
    flag_debug: bool,
    flag_benchmark: bool,
}

#[cfg(test)]
mod tests {

    use std::collections::HashMap;
    use ::AggregateELBRecord;
    use ::aggregate_record;

    const TEST_RECORD: &'static str = "2015-08-15T23:43:05.302180Z elb-name 172.16.1.6:54814 \
    172.16.1.5:9000 0.000039 0.145507 0.00003 200 200 0 7582 \
    \"GET http://some.domain.com:80/path0/path1?param0=p0&param1=p1 HTTP/1.1\"\
    ";

    #[test]
	fn inserting_two_records_with_different_values_creates_two_entries_each_recorded_once() {
        let mut agg: HashMap<AggregateELBRecord, i64> = HashMap::new();

        let ar0 = AggregateELBRecord {
            day: "2015-08-15".to_owned(),
            client_address: "172.16.1.6:54814".to_owned(),
            system_name: "sys1".to_owned(),
            public_id: 8880
        };

        let ar1 = AggregateELBRecord {
            day: "2015-08-15".to_owned(),
            client_address: "172.16.1.6:54814".to_owned(),
            system_name: "sys1".to_owned(),
            public_id: 8888
        };

        aggregate_record(ar0, &mut agg);
        aggregate_record(ar1, &mut agg);

        assert_eq!(agg.len(), 2);
        for (_, total) in agg {
            assert_eq!(total, 1)
        }
	}

    #[test]
	fn inserting_two_records_with_the_same_values_increases_the_total_correctly() {
        let mut agg: HashMap<AggregateELBRecord, i64> = HashMap::new();

        let ar0 = AggregateELBRecord {
            day: "2015-08-15".to_owned(),
            client_address: "172.16.1.6:54814".to_owned(),
            system_name: "sys1".to_owned(),
            public_id: 8888
        };

        let ar1 = ar0.clone();
        let ar3 = ar0.clone();

        aggregate_record(ar0, &mut agg);
        aggregate_record(ar1, &mut agg);

        assert_eq!(agg[&ar3], 2);
	}
}
<commit_msg>Rewrote the parse_system_name function in a cleaner functional style.<commit_after>extern crate rustc_serialize;
#[macro_use]
extern crate aws_abacus;
#[macro_use]
extern crate log;
extern crate walkdir;
extern crate env_logger;
extern crate docopt;
use docopt::Docopt;
use std::path;
use aws_abacus::elb_log_files;
extern crate chrono;
use chrono::{DateTime, UTC};
use aws_abacus::elb_log_files::ParsingResult;
use std::collections::HashMap;
extern crate urlparse;
use urlparse::{Url, urlparse};

fn main() {
    env_logger::init().unwrap();
    let args: Args = Docopt::new(USAGE)
                            .and_then(|d| d.decode())
                            .unwrap_or_else(|e| e.exit());

    let log_location = &path::Path::new(&args.arg_log_location);
    debug!("Running summary on {}.", log_location.to_str().unwrap());

    let start: Option<DateTime<UTC>> = if args.flag_benchmark {
        Some(UTC::now())
    } else {
        None
    };

    let mut number_of_files = 0;
    let mut number_of_records = 0;
    let mut filenames = Vec::new();
    let mut agg: HashMap<AggregateELBRecord, i64> = HashMap::new();
    match elb_log_files::file_list(log_location, &mut filenames) {
        Ok(num_files) => {
            number_of_files = num_files;
            debug!("Found {} files.", number_of_files);
            number_of_records = elb_log_files::process_files(&filenames, &mut |parsing_result: ParsingResult| {
                parsing_result_handler(parsing_result, &mut agg);
            });
            debug!("Processed {} records in {} files.", number_of_records, num_files);
        },
        Err(e) => {
            println!("ERROR: {}", e);
        },
    };

    for (aggregate, total) in &agg {
        println!(
            "{},{},{},{}",
            aggregate.system_name,
            aggregate.day,
            aggregate.client_address,
            total
        );
    };

    match start {
        Some(s) => {
            let end = UTC::now();
            let time = end - s;
            println!("Processed {} files having {} records in {} milliseconds and produced {} aggregates.",
                number_of_files,
                number_of_records,
                time.num_milliseconds(),
                agg.len()
            );
        },
        None => {},
    };
}

#[derive(Clone, Debug, Eq, Hash, PartialEq)]
struct AggregateELBRecord {
    day: String,
    client_address: String,
    system_name: String,
}

fn parsing_result_handler(parsing_result: ParsingResult, aggregation: &mut HashMap<AggregateELBRecord, i64>) -> () {
    match parsing_result {
        Ok(elb_record) => {
            let url = urlparse(&elb_record.request_url);
            let aer = AggregateELBRecord {
                day: elb_record.timestamp.format("%Y-%m-%d").to_string(),
                client_address: elb_record.client_address.ip().to_string(),
                system_name: parse_system_name(&url).unwrap_or("UNDEFINED_SYSTEM".to_owned()),
            };
            aggregate_record(aer, aggregation);
        },

        Err(errors) => {

        }
    }
}

fn parse_system_name(url: &Url) -> Option<String> {
    url.get_parsed_query().map( |query_map|
        query_map.get("system").map( |systems|
            systems[0].clone()
        )
    ).unwrap_or_else( || None )
}

fn aggregate_record(aggregate_record: AggregateELBRecord, aggregation: &mut HashMap<AggregateELBRecord, i64>) -> () {
    let total = aggregation.entry(aggregate_record).or_insert(0);
    *total += 1;
}

const USAGE: &'static str = "
aws-abacus

Usage:
  aws-abacus <log-location>
  aws-abacus (-d | --debug | -b | --benchmark) <log-location>
  aws-abacus (-h | --help)

Options:
  -h --help     Show this screen.
  -d --debug    Turn on debug output
";

#[derive(Debug, RustcDecodable)]
struct Args {
    arg_log_location: String,
    flag_debug: bool,
    flag_benchmark: bool,
}

#[cfg(test)]
mod tests {

    use std::collections::HashMap;
    use ::AggregateELBRecord;
    use ::aggregate_record;

    const TEST_RECORD: &'static str = "2015-08-15T23:43:05.302180Z elb-name 172.16.1.6:54814 \
    172.16.1.5:9000 0.000039 0.145507 0.00003 200 200 0 7582 \
    \"GET http://some.domain.com:80/path0/path1?param0=p0&param1=p1 HTTP/1.1\"\
    ";

    #[test]
	fn inserting_two_records_with_different_values_creates_two_entries_each_recorded_once() {
        let mut agg: HashMap<AggregateELBRecord, i64> = HashMap::new();

        let ar0 = AggregateELBRecord {
            day: "2015-08-15".to_owned(),
            client_address: "172.16.1.6:54814".to_owned(),
            system_name: "sys1".to_owned(),
            public_id: 8880
        };

        let ar1 = AggregateELBRecord {
            day: "2015-08-15".to_owned(),
            client_address: "172.16.1.6:54814".to_owned(),
            system_name: "sys1".to_owned(),
            public_id: 8888
        };

        aggregate_record(ar0, &mut agg);
        aggregate_record(ar1, &mut agg);

        assert_eq!(agg.len(), 2);
        for (_, total) in agg {
            assert_eq!(total, 1)
        }
	}

    #[test]
	fn inserting_two_records_with_the_same_values_increases_the_total_correctly() {
        let mut agg: HashMap<AggregateELBRecord, i64> = HashMap::new();

        let ar0 = AggregateELBRecord {
            day: "2015-08-15".to_owned(),
            client_address: "172.16.1.6:54814".to_owned(),
            system_name: "sys1".to_owned(),
            public_id: 8888
        };

        let ar1 = ar0.clone();
        let ar3 = ar0.clone();

        aggregate_record(ar0, &mut agg);
        aggregate_record(ar1, &mut agg);

        assert_eq!(agg[&ar3], 2);
	}
}
<|endoftext|>