<commit_before>extern crate docopt;
#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate error_chain;

use docopt::Docopt;
use std::fs::File;
use std::io::{Read, Write, stdin, stdout, stderr};

mod errors {
    error_chain!{
        foreign_links {
            Io(::std::io::Error);
        }
    }
}

use errors::*;

const USAGE: &str = "
hexlify 0.0.1

Perform bytes-to-hexstring conversion and vice-versa as implemented
in Python's binascii.{un,}hexlify. Read from stdin if <file> is \"-\"
or not specified. Whitespace is ignored during decoding.

Usage: hexlify [options] [<file>]

Options:
  -d --decode          Decode stream.
  -i --ignore-garbage  Ignore non-hex values.
  -h --help            Show this help screen.
";

const ERR_NOT_IN_HEX: &str = "\
Character does not match hex-alphabet. Only 0-9, a-f and A-F are allowed.
Make sure not to confuse decoding with encoding or use -i to ignore non-hex characters.\
";

const ERR_ODD_CHARS: &str = "\
Input had odd number of characters, please be cautious.\
";

#[derive(Deserialize)]
struct Args {
    arg_file: Option<String>,
    flag_decode: bool,
    flag_ignore_garbage: bool,
}

fn to_hex_value(c: u8) -> Result<u8> {
    match c as char {
        '0'...'9' => Ok(c - '0' as u8),
        'a'...'f' => Ok(c - 'a' as u8 + 10),
        'A'...'F' => Ok(c - 'A' as u8 + 10),
        _ => Err(ERR_NOT_IN_HEX.into()),
    }
}

fn pair_to_hex(pair: &[u8; 2]) -> Result<u8> {
    let l = to_hex_value(pair[0])?;
    let r = to_hex_value(pair[1])?;
    Ok(l * 16 + r)
}

fn decode<R: Read, W: Write>(src: &mut R, dst: &mut W, ignore_garbage: bool) -> Result<()> {
    let mut side = 0;
    let mut pair = [0 as u8; 2];

    for byte in src.bytes() {
        let byte = byte?;

        if (byte as char).is_whitespace() {
            continue;
        }

        if to_hex_value(byte).is_err() && ignore_garbage {
            continue;
        }

        pair[side] = byte;

        if side == 0 {
            side = 1;
        } else {
            dst.write(&[pair_to_hex(&pair)?])?;
            side = 0;
        }
    }

    if side == 1 {
        return Err(ERR_ODD_CHARS.into());
    }

    Ok(())
}

fn encode<R: Read, W: Write>(src: &mut R, dst: &mut W) -> Result<()> {
    for byte in src.bytes() {
        write!(dst, "{:02X}", byte?)?;
    }

    Ok(())
}

fn run(file: Option<String>, flag_decode: bool, flag_ignore_garbage: bool) -> Result<()> {
    // Choose input (file or stdin)
    let mut src: Box<Read> = match file {
        Some(path) => {
            if path != "-" {
                Box::new(File::open(&path).chain_err(|| format!("can't open \"{}\"", path))?)
            } else {
                Box::new(stdin())
            }
        }
        None => Box::new(stdin()),
    };

    // Choose output (stdout)
    let mut dst = stdout();

    if flag_decode {
        decode(&mut src, &mut dst, flag_ignore_garbage)?;
    } else {
        encode(&mut src, &mut dst)?;
    }

    Ok(())
}

fn main() {
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| d.deserialize())
        .unwrap_or_else(|e| e.exit());

    if let Err(ref e) = run(args.arg_file, args.flag_decode, args.flag_ignore_garbage) {
        writeln!(stderr(), "error: {}", e).unwrap();

        for e in e.iter().skip(1) {
            writeln!(stderr(), "caused by: {}", e).unwrap();
        }

        if let Some(backtrace) = e.backtrace() {
            writeln!(stderr(), "backtrace: {:?}", backtrace).unwrap();
        }

        ::std::process::exit(1);
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn test_to_hex_value() {
        assert_eq!(to_hex_value('0' as u8).unwrap(), 0);
        assert_eq!(to_hex_value('9' as u8).unwrap(), 9);
        assert_eq!(to_hex_value('a' as u8).unwrap(), 10);
        assert_eq!(to_hex_value('f' as u8).unwrap(), 15);
        assert_eq!(to_hex_value('A' as u8).unwrap(), 10);
        assert_eq!(to_hex_value('F' as u8).unwrap(), 15);
    }

    #[test]
    fn test_pair_to_hex() {
        assert_eq!(pair_to_hex(&['0' as u8, '0' as u8]).unwrap(), 0x00);
        assert_eq!(pair_to_hex(&['9' as u8, '9' as u8]).unwrap(), 0x99);
        assert_eq!(pair_to_hex(&['a' as u8, 'a' as u8]).unwrap(), 0xAA);
        assert_eq!(pair_to_hex(&['A' as u8, 'A' as u8]).unwrap(), 0xAA);
        assert_eq!(pair_to_hex(&['f' as u8, 'f' as u8]).unwrap(), 0xFF);
        assert_eq!(pair_to_hex(&['F' as u8, 'F' as u8]).unwrap(), 0xFF);

        assert!(pair_to_hex(&['0' as u8, '/' as u8]).is_err());
        assert!(pair_to_hex(&['0' as u8, ':' as u8]).is_err());

        assert!(pair_to_hex(&['0' as u8, '`' as u8]).is_err());
        assert!(pair_to_hex(&['0' as u8, 'g' as u8]).is_err());

        assert!(pair_to_hex(&['0' as u8, '@' as u8]).is_err());
        assert!(pair_to_hex(&['0' as u8, 'G' as u8]).is_err());
    }

    #[test]
    fn test_decode() {
        let samples = vec![
            ("AA", vec![0xAA]),
            ("AABB", vec![0xAA, 0xBB]),
            ("AABBCCD", vec![0xAA, 0xBB, 0xCC]),
            ("AABBCCDD", vec![0xAA, 0xBB, 0xCC, 0xDD]),
            ("00gf", vec![0x00]),
            ("00g  ff", vec![0x00, 0xff]),
            ("---f-..,f aa b", vec![0xff, 0xaa]),
            ("²¹²³fa„“", vec![0xfa]),
            ("¹¸^afg01", vec![0xaf, 0x01]),
            ("´#*-_ff:;:;:1234", vec![0xff, 0x12, 0x34]),
            ("", vec![]),
            ("a1¹²}ff™± ¡¿⅛°±£™⅞`ff¿¡⅛°±£™01`", vec![0xa1, 0xff, 0xff, 0x01]),
        ];

        for (mut sample, expected) in samples.into_iter().map(|(s, e)| (std::io::Cursor::new(s), e)) {
            let got = {
                let mut tmp = Vec::new();
                let _ = decode(&mut sample, &mut tmp, true);
                tmp
            };

            assert_eq!(expected, got);
        }
    }

    #[test]
    fn test_encode() {
        let samples = vec![
            (vec![], ""),
            (vec![0x00], "00"),
            (vec![0x00, 0x99, 0xAA, 0xFF], "0099AAFF"),
            (vec![0x00, 0x99, 0xAA, 0xFF, 0x00, 0x00], "0099AAFF0000"),
        ];

        for (mut sample, expected) in samples.into_iter().map(|(s, e)| (std::io::Cursor::new(s), e)) {
            let got = {
                let mut tmp = Vec::new();
                let _ = encode(&mut sample, &mut tmp);
                String::from_utf8(tmp).unwrap()
            };

            assert_eq!(expected, got);
        }
    }
}
<commit_msg>Take version from Cargo.toml<commit_after>extern crate docopt;
#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate error_chain;

use docopt::Docopt;
use std::fs::File;
use std::io::{Read, Write, stdin, stdout, stderr};

mod errors {
    error_chain!{
        foreign_links {
            Io(::std::io::Error);
        }
    }
}

use errors::*;

const USAGE: &str = "
hexlify

Perform bytes-to-hexstring conversion and vice-versa as implemented
in Python's binascii.{un,}hexlify. Read from stdin if <file> is \"-\"
or not specified. Whitespace is ignored during decoding.

Usage: hexlify [options] [<file>]

Options:
  -d --decode          Decode stream.
  -i --ignore-garbage  Ignore non-hex values.
  -h --help            Show this help screen.
";

const ERR_NOT_IN_HEX: &str = "\
Character does not match hex-alphabet. Only 0-9, a-f and A-F are allowed.
Make sure not to confuse decoding with encoding or use -i to ignore non-hex characters.\
";

const ERR_ODD_CHARS: &str = "\
Input had odd number of characters, please be cautious.\
";

// Get version from Cargo.toml
const VERSION: &str = env!("CARGO_PKG_VERSION");

#[derive(Deserialize)]
struct Args {
    arg_file: Option<String>,
    flag_decode: bool,
    flag_ignore_garbage: bool,
}

fn to_hex_value(c: u8) -> Result<u8> {
    match c as char {
        '0'...'9' => Ok(c - '0' as u8),
        'a'...'f' => Ok(c - 'a' as u8 + 10),
        'A'...'F' => Ok(c - 'A' as u8 + 10),
        _ => Err(ERR_NOT_IN_HEX.into()),
    }
}

fn pair_to_hex(pair: &[u8; 2]) -> Result<u8> {
    let l = to_hex_value(pair[0])?;
    let r = to_hex_value(pair[1])?;
    Ok(l * 16 + r)
}

fn decode<R: Read, W: Write>(src: &mut R, dst: &mut W, ignore_garbage: bool) -> Result<()> {
    let mut side = 0;
    let mut pair = [0 as u8; 2];

    for byte in src.bytes() {
        let byte = byte?;

        if (byte as char).is_whitespace() {
            continue;
        }

        if to_hex_value(byte).is_err() && ignore_garbage {
            continue;
        }

        pair[side] = byte;

        if side == 0 {
            side = 1;
        } else {
            dst.write(&[pair_to_hex(&pair)?])?;
            side = 0;
        }
    }

    if side == 1 {
        return Err(ERR_ODD_CHARS.into());
    }

    Ok(())
}

fn encode<R: Read, W: Write>(src: &mut R, dst: &mut W) -> Result<()> {
    for byte in src.bytes() {
        write!(dst, "{:02X}", byte?)?;
    }

    Ok(())
}

fn run(file: Option<String>, flag_decode: bool, flag_ignore_garbage: bool) -> Result<()> {
    // Choose input (file or stdin)
    let mut src: Box<Read> = match file {
        Some(path) => {
            if path != "-" {
                Box::new(File::open(&path).chain_err(|| format!("can't open \"{}\"", path))?)
            } else {
                Box::new(stdin())
            }
        }
        None => Box::new(stdin()),
    };

    // Choose output (stdout)
    let mut dst = stdout();

    if flag_decode {
        decode(&mut src, &mut dst, flag_ignore_garbage)?;
    } else {
        encode(&mut src, &mut dst)?;
    }

    Ok(())
}

fn main() {
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| d.version(Some(VERSION.into())).deserialize())
        .unwrap_or_else(|e| e.exit());

    if let Err(ref e) = run(args.arg_file, args.flag_decode, args.flag_ignore_garbage) {
        writeln!(stderr(), "error: {}", e).unwrap();

        for e in e.iter().skip(1) {
            writeln!(stderr(), "caused by: {}", e).unwrap();
        }

        if let Some(backtrace) = e.backtrace() {
            writeln!(stderr(), "backtrace: {:?}", backtrace).unwrap();
        }

        ::std::process::exit(1);
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn test_to_hex_value() {
        assert_eq!(to_hex_value('0' as u8).unwrap(), 0);
        assert_eq!(to_hex_value('9' as u8).unwrap(), 9);
        assert_eq!(to_hex_value('a' as u8).unwrap(), 10);
        assert_eq!(to_hex_value('f' as u8).unwrap(), 15);
        assert_eq!(to_hex_value('A' as u8).unwrap(), 10);
        assert_eq!(to_hex_value('F' as u8).unwrap(), 15);
    }

    #[test]
    fn test_pair_to_hex() {
        assert_eq!(pair_to_hex(&['0' as u8, '0' as u8]).unwrap(), 0x00);
        assert_eq!(pair_to_hex(&['9' as u8, '9' as u8]).unwrap(), 0x99);
        assert_eq!(pair_to_hex(&['a' as u8, 'a' as u8]).unwrap(), 0xAA);
        assert_eq!(pair_to_hex(&['A' as u8, 'A' as u8]).unwrap(), 0xAA);
        assert_eq!(pair_to_hex(&['f' as u8, 'f' as u8]).unwrap(), 0xFF);
        assert_eq!(pair_to_hex(&['F' as u8, 'F' as u8]).unwrap(), 0xFF);

        assert!(pair_to_hex(&['0' as u8, '/' as u8]).is_err());
        assert!(pair_to_hex(&['0' as u8, ':' as u8]).is_err());

        assert!(pair_to_hex(&['0' as u8, '`' as u8]).is_err());
        assert!(pair_to_hex(&['0' as u8, 'g' as u8]).is_err());

        assert!(pair_to_hex(&['0' as u8, '@' as u8]).is_err());
        assert!(pair_to_hex(&['0' as u8, 'G' as u8]).is_err());
    }

    #[test]
    fn test_decode() {
        let samples = vec![
            ("AA", vec![0xAA]),
            ("AABB", vec![0xAA, 0xBB]),
            ("AABBCCD", vec![0xAA, 0xBB, 0xCC]),
            ("AABBCCDD", vec![0xAA, 0xBB, 0xCC, 0xDD]),
            ("00gf", vec![0x00]),
            ("00g  ff", vec![0x00, 0xff]),
            ("---f-..,f aa b", vec![0xff, 0xaa]),
            ("²¹²³fa„“", vec![0xfa]),
            ("¹¸^afg01", vec![0xaf, 0x01]),
            ("´#*-_ff:;:;:1234", vec![0xff, 0x12, 0x34]),
            ("", vec![]),
            ("a1¹²}ff™± ¡¿⅛°±£™⅞`ff¿¡⅛°±£™01`", vec![0xa1, 0xff, 0xff, 0x01]),
        ];

        for (mut sample, expected) in samples.into_iter().map(|(s, e)| (std::io::Cursor::new(s), e)) {
            let got = {
                let mut tmp = Vec::new();
                let _ = decode(&mut sample, &mut tmp, true);
                tmp
            };

            assert_eq!(expected, got);
        }
    }

    #[test]
    fn test_encode() {
        let samples = vec![
            (vec![], ""),
            (vec![0x00], "00"),
            (vec![0x00, 0x99, 0xAA, 0xFF], "0099AAFF"),
            (vec![0x00, 0x99, 0xAA, 0xFF, 0x00, 0x00], "0099AAFF0000"),
        ];

        for (mut sample, expected) in samples.into_iter().map(|(s, e)| (std::io::Cursor::new(s), e)) {
            let got = {
                let mut tmp = Vec::new();
                let _ = encode(&mut sample, &mut tmp);
                String::from_utf8(tmp).unwrap()
            };

            assert_eq!(expected, got);
        }
    }
}
<|endoftext|>


<commit_before>extern crate mmap;
extern crate time;
extern crate timely;
extern crate getopts;

extern crate graph_map;
extern crate graph_layout;
extern crate radix_sort;

use timely::progress::timestamp::RootTimestamp;
use timely::progress::nested::Summary::Local;
use timely::dataflow::*;
use timely::dataflow::operators::*;
use timely::dataflow::channels::pact::{Pipeline, Exchange};
use timely::drain::DrainExt;

use graph_layout::layout::*;
use graph_layout::compression::*;

use graph_map::graph_map::GraphMMap;

use radix_sort::*;

fn main () {

    /*
        The plan here is to partition the rows of the matrix into #processes parts, and the columns
        into #threads parts. This could be generally #row_parts and #col_parts, but that requires
        more thinking about things.
    */

    let threads: u32 = ::std::env::args().nth(1).unwrap().parse().unwrap();
    let filename = ::std::env::args().nth(2).unwrap();
    let nodes: u32 = ::std::env::args().nth(3).unwrap().parse().unwrap();

    timely::execute_from_args(::std::env::args().skip(4), move |root| {

        let index = root.index() as u32;
        let peers = root.peers() as u32;
        let processes = (peers / threads) as u32;
        let dst_index = index % threads;

        let start = time::precise_time_s();
        let mut going = start;

        let mut counter = 0;
        let mut sorter = RadixSorter::new();
        let hilbert = Hilbert::new();
        let mut decoder = BytewiseCached::new();
        let mut compressed = vec![];

        // let nodes: u32 = 3_563_602_788 + 1;

        let local_srcs = (nodes / processes) + 1;
        let local_dsts = (nodes / threads) + 1;

        // degree is ignored at the moment.
        // mostly just a pain to put together.
        let deg = vec![1.0f32; local_srcs as usize];
        let mut src = vec![1.0f32; local_srcs as usize];
        let mut dst = vec![0.0f32; local_dsts as usize];

        let mut input = root.scoped(|builder| {

            let (input, edges) = builder.new_input::<(u32, u32)>();
            let (cycle, ranks) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

            let ranks = edges.binary_notify(&ranks,
                                Exchange::new(|x: &(u32,u32)| ((x.0 % machines) * threads + (x.1 % threads)) as u64),
                                Pipeline,   // data are partitioned as part of the broadcast step.
                                "hilbrank",
                                vec![RootTimestamp::new(0)],
                                move |input1, input2, output, notificator| {

                // receive incoming edges (should only be iter 0)
                while let Some((_index, data)) = input1.next() {
                    counter += data.len();
                    for &(src,dst) in data.iter() {
                        let shrunk = (src / processes, dst / threads);
                        sorter.push(hilbert.entangle(shrunk), &|&x| x);
                    }

                    // should be 1GB per worker
                    if counter > (1 << 27) {
                        let mut results = sorter.finish(&|&x| x);
                        let mut compressor = Compressor::with_capacity(counter);
                        for vec in results.iter_mut() {
                            for elem in vec.drain_temp() {
                                compressor.push(elem);
                            }
                        }
                        sorter.recycle(results);
                        compressed.push(compressor.done());
                        counter = 0;
                    }
                }

                if counter > 0 {
                    // round out any remaining edges
                    let mut results = sorter.finish(&|&x| x);
                    let mut compressor = Compressor::with_capacity(counter);
                    for vec in results.iter_mut() {
                        for elem in vec.drain_temp() {
                            compressor.push(elem);
                        }
                    }
                    compressed.push(compressor.done());
                    counter = 0;
                }

                // all inputs received for iter, commence multiplication
                while let Some((iter, _)) = notificator.next() {

                    let now = time::now();

                    if index == 0 { println!("{}:{}:{}.{} starting iteration {}", now.tm_hour, now.tm_min, now.tm_sec, now.tm_nsec, iter.inner); }

                    // if the very first iteration, prepare some stuff.
                    if iter.inner == 0 {
                        // now we would merge the compressed representations, if we want to.
                        // let's measure both doing this and not doing it. :)

                    }

                    // record some timings in order to estimate per-iteration times
                    // if iter.inner == 0  && index == 0 { println!("src: {}, dst: {}, edges: {}", src.len(), rev.len(), trn.len()); }
                    if iter.inner == 10 && index == 0 { going = time::precise_time_s(); }
                    if iter.inner == 20 && index == 0 { println!("average: {}", (time::precise_time_s() - going) / 10.0 ); }

                    // prepare src for transmitting to destinations
                    for s in 0..src.len() { src[s] = 0.15 + 0.85 * src[s] / deg[s] as f32; }
                    for d in 0..dst.len() { dst[d] = 0.0; }

                    // do the multiplication for each compressed sequence
                    for sequence in &compressed {
                        for element in sequence.decompress() {
                            let (s,d) = decoder.detangle(element);
                            dst[d as usize] += src[s as usize];
                        }
                    }

                    for s in 0..src.len() { src[s] = 0.0; }

                    // now we need to send some stuff to other threads.
                    // TODO : This could be done as blocks, as they mostly go to the same place.
                    let mut session = output.session(&iter);
                    for (index, &value) in dst.iter().enumerate() {
                        if value != 0.0 {
                            session.give((index as u32 * threads + dst_index, value));
                        }
                    }
                }

                // receive data from threads, accumulate in src
                while let Some((iter, data)) = input2.next() {
                    notificator.notify_at(&iter);
                    for &(node, rank) in data.iter() {
                        src[(node / processes) as usize] += rank;
                    }
                }
            });

            let mut aggregates = vec![0.0f32; (nodes / peers) as usize];
            // want to do aggregation so that aggregates are local to the folks who will need them.
            // that is, (node, rank) goes to some worker on machine#: name % processes
            let aggregated = ranks.unary_notify(
                Exchange::new(move |&(n,_): &(u32,f32)| (((n % processes) * threads) + ((n / processes) % threads)) as u64),
                "aggregation",
                vec![],
                move |input, output, notificator | {

                while let Some((time, data)) = input.next() {
                    notificator.notify_at(time);
                    for &(node, rank) in data.iter() {
                        aggregates[(node / peers) as usize] += rank;
                    }
                }

                while let Some((time, _)) = notificator.next() {
                    let mut session = output.session(&time);
                    for (node, &rank) in aggregates.iter().enumerate() {
                        if rank != 0.0 {
                            session.give((node as u32 * peers, rank));
                        }
                    }
                }
            });

            // data are now aggregated, and only need to be broadcast out to the intended recipients
            // ideally, none of this actually ends up hitting the network. also, it seems like it
            // shouldn't even require looking at `node`, on account of already putting all the data
            // on the right processes. We should be able to use (index / threads) * threads, I think.
            let mut exchanged = aggregated.exchange(move |&(node, _)| ((node % processes) * threads) as u64);
            for i in 1..threads {
                exchanged = aggregated.exchange(move |&(node, _)| (((node % processes) * threads) + i) as u64)
                                      .concat(&exchanged);
            }

            exchanged.connect_loop(cycle);

            input
        });

        // introduce edges into the computation;
        // allow mmaped file to drop
        {
            let graph = GraphMMap::new(&filename);
            for node in 0..graph.nodes() {
                if node as u32 % peers == index {
                    for dst in graph.edges(node) {
                        input.send((node as u32, *dst as u32));
                    }
                }
            }
        }
        input.close();
        while root.step() { }

        if index == 0 { println!("elapsed: {}", time::precise_time_s() - start); }
    });

}
<commit_msg>lolwoops<commit_after>extern crate mmap;
extern crate time;
extern crate timely;
extern crate getopts;

extern crate graph_map;
extern crate graph_layout;
extern crate radix_sort;

use timely::progress::timestamp::RootTimestamp;
use timely::progress::nested::Summary::Local;
use timely::dataflow::*;
use timely::dataflow::operators::*;
use timely::dataflow::channels::pact::{Pipeline, Exchange};
use timely::drain::DrainExt;

use graph_layout::layout::*;
use graph_layout::compression::*;

use graph_map::graph_map::GraphMMap;

use radix_sort::*;

fn main () {

    /*
        The plan here is to partition the rows of the matrix into #processes parts, and the columns
        into #threads parts. This could be generally #row_parts and #col_parts, but that requires
        more thinking about things.
    */

    let threads: u32 = ::std::env::args().nth(1).unwrap().parse().unwrap();
    let filename = ::std::env::args().nth(2).unwrap();
    let nodes: u32 = ::std::env::args().nth(3).unwrap().parse().unwrap();

    timely::execute_from_args(::std::env::args().skip(4), move |root| {

        let index = root.index() as u32;
        let peers = root.peers() as u32;
        let processes = (peers / threads) as u32;
        let dst_index = index % threads;

        let start = time::precise_time_s();
        let mut going = start;

        let mut counter = 0;
        let mut sorter = RadixSorter::new();
        let hilbert = Hilbert::new();
        let mut decoder = BytewiseCached::new();
        let mut compressed = vec![];

        // let nodes: u32 = 3_563_602_788 + 1;

        let local_srcs = (nodes / processes) + 1;
        let local_dsts = (nodes / threads) + 1;

        // degree is ignored at the moment.
        // mostly just a pain to put together.
        let deg = vec![1.0f32; local_srcs as usize];
        let mut src = vec![1.0f32; local_srcs as usize];
        let mut dst = vec![0.0f32; local_dsts as usize];

        let mut input = root.scoped(|builder| {

            let (input, edges) = builder.new_input::<(u32, u32)>();
            let (cycle, ranks) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

            let ranks = edges.binary_notify(&ranks,
                                Exchange::new(|x: &(u32,u32)| ((x.0 % processes) * threads + (x.1 % threads)) as u64),
                                Pipeline,   // data are partitioned as part of the broadcast step.
                                "hilbrank",
                                vec![RootTimestamp::new(0)],
                                move |input1, input2, output, notificator| {

                // receive incoming edges (should only be iter 0)
                while let Some((_index, data)) = input1.next() {
                    counter += data.len();
                    for &(src,dst) in data.iter() {
                        let shrunk = (src / processes, dst / threads);
                        sorter.push(hilbert.entangle(shrunk), &|&x| x);
                    }

                    // should be 1GB per worker
                    if counter > (1 << 27) {
                        let mut results = sorter.finish(&|&x| x);
                        let mut compressor = Compressor::with_capacity(counter);
                        for vec in results.iter_mut() {
                            for elem in vec.drain_temp() {
                                compressor.push(elem);
                            }
                        }
                        sorter.recycle(results);
                        compressed.push(compressor.done());
                        counter = 0;
                    }
                }

                if counter > 0 {
                    // round out any remaining edges
                    let mut results = sorter.finish(&|&x| x);
                    let mut compressor = Compressor::with_capacity(counter);
                    for vec in results.iter_mut() {
                        for elem in vec.drain_temp() {
                            compressor.push(elem);
                        }
                    }
                    compressed.push(compressor.done());
                    counter = 0;
                }

                // all inputs received for iter, commence multiplication
                while let Some((iter, _)) = notificator.next() {

                    let now = time::now();

                    if index == 0 { println!("{}:{}:{}.{} starting iteration {}", now.tm_hour, now.tm_min, now.tm_sec, now.tm_nsec, iter.inner); }

                    // if the very first iteration, prepare some stuff.
                    if iter.inner == 0 {
                        // now we would merge the compressed representations, if we want to.
                        // let's measure both doing this and not doing it. :)

                    }

                    // record some timings in order to estimate per-iteration times
                    // if iter.inner == 0  && index == 0 { println!("src: {}, dst: {}, edges: {}", src.len(), rev.len(), trn.len()); }
                    if iter.inner == 10 && index == 0 { going = time::precise_time_s(); }
                    if iter.inner == 20 && index == 0 { println!("average: {}", (time::precise_time_s() - going) / 10.0 ); }

                    // prepare src for transmitting to destinations
                    for s in 0..src.len() { src[s] = 0.15 + 0.85 * src[s] / deg[s] as f32; }
                    for d in 0..dst.len() { dst[d] = 0.0; }

                    // do the multiplication for each compressed sequence
                    for sequence in &compressed {
                        for element in sequence.decompress() {
                            let (s,d) = decoder.detangle(element);
                            dst[d as usize] += src[s as usize];
                        }
                    }

                    for s in 0..src.len() { src[s] = 0.0; }

                    // now we need to send some stuff to other threads.
                    // TODO : This could be done as blocks, as they mostly go to the same place.
                    let mut session = output.session(&iter);
                    for (index, &value) in dst.iter().enumerate() {
                        if value != 0.0 {
                            session.give((index as u32 * threads + dst_index, value));
                        }
                    }
                }

                // receive data from threads, accumulate in src
                while let Some((iter, data)) = input2.next() {
                    notificator.notify_at(&iter);
                    for &(node, rank) in data.iter() {
                        src[(node / processes) as usize] += rank;
                    }
                }
            });

            let mut aggregates = vec![0.0f32; (nodes / peers) as usize];
            // want to do aggregation so that aggregates are local to the folks who will need them.
            // that is, (node, rank) goes to some worker on machine#: name % processes
            let aggregated = ranks.unary_notify(
                Exchange::new(move |&(n,_): &(u32,f32)| (((n % processes) * threads) + ((n / processes) % threads)) as u64),
                "aggregation",
                vec![],
                move |input, output, notificator | {

                while let Some((time, data)) = input.next() {
                    notificator.notify_at(time);
                    for &(node, rank) in data.iter() {
                        aggregates[(node / peers) as usize] += rank;
                    }
                }

                while let Some((time, _)) = notificator.next() {
                    let mut session = output.session(&time);
                    for (node, &rank) in aggregates.iter().enumerate() {
                        if rank != 0.0 {
                            session.give((node as u32 * peers, rank));
                        }
                    }
                }
            });

            // data are now aggregated, and only need to be broadcast out to the intended recipients
            // ideally, none of this actually ends up hitting the network. also, it seems like it
            // shouldn't even require looking at `node`, on account of already putting all the data
            // on the right processes. We should be able to use (index / threads) * threads, I think.
            let mut exchanged = aggregated.exchange(move |&(node, _)| ((node % processes) * threads) as u64);
            for i in 1..threads {
                exchanged = aggregated.exchange(move |&(node, _)| (((node % processes) * threads) + i) as u64)
                                      .concat(&exchanged);
            }

            exchanged.connect_loop(cycle);

            input
        });

        // introduce edges into the computation;
        // allow mmaped file to drop
        {
            let graph = GraphMMap::new(&filename);
            for node in 0..graph.nodes() {
                if node as u32 % peers == index {
                    for dst in graph.edges(node) {
                        input.send((node as u32, *dst as u32));
                    }
                }
            }
        }
        input.close();
        while root.step() { }

        if index == 0 { println!("elapsed: {}", time::precise_time_s() - start); }
    });

}
<|endoftext|>


<commit_before>extern crate bindgen;
#[macro_use]
extern crate log;
extern crate docopt;
#[macro_use]
extern crate rustc_serialize;
extern crate env_logger;

use bindgen::{Builder, LinkType};
use std::io::{self, Write};
use std::fs::File;
use std::process::exit;

const USAGE: &'static str = "
Generate C bindings for Rust.

Usage:
  bindgen [options] <file> [-- <clang-args>...]
  bindgen (-h | --help)

Options:
  <clang-args>                 Options passed directly to clang.
  -h, --help                   Display this help message.
  --link=<library>             Link to a dynamic library, can be provided multiple times.
                               <library> is in the format `[kind=]lib`, where `kind` is
                               one of `static`, `dynamic` or `framework`.
  --output=<output>            Write bindings to <output> (- is stdout).
                               [default: -]
  --match=<name>               Only output bindings for definitions from files
                               whose name contains <name>
                               If multiple -match options are provided, files
                               matching any rule are bound to.
  --builtins                   Output bindings for builtin definitions
                               (for example __builtin_va_list)
  --emit-clang-ast             Output the ast (for debugging purposes)
  --override-enum-type=<type>  Override enum type, type name could be
                                 uchar
                                 schar
                                 ushort
                                 sshort
                                 uint
                                 sint
                                 ulong
                                 slong
                                 ulonglong
                                 slonglong
  --use-core                  Use `core` as a base crate for `Option` and such.
                              See also `--ctypes-prefix`.
  --ctypes-prefix=<prefix>    Use this prefix for all the types in the generated
                              code.
                              [default: std::os::raw]
  --remove-prefix=<prefix>    Prefix to remove from all the symbols, like
                              `libfoo_`. The removal is case-insensitive.
";

#[derive(Debug, RustcDecodable)]
struct Args {
    arg_file: String,
    arg_clang_args: Vec<String>,
    flag_link: Option<String>,
    flag_output: String,
    flag_match: Option<String>,
    flag_builtins: bool,
    flag_emit_clang_ast: bool,
    flag_override_enum_type: String,
    flag_ctypes_prefix: String,
    flag_use_core: bool,
    flag_remove_prefix: Option<String>,
}

fn args_to_opts(args: Args, builder: &mut Builder) {
    builder.header(args.arg_file)
           .emit_ast(args.flag_emit_clang_ast)
           .ctypes_prefix(args.flag_ctypes_prefix
                              .split("::")
                              .map(String::from)
                              .collect::<Vec<_>>())
           .use_core(args.flag_use_core)
           .override_enum_ty(args.flag_override_enum_type);
    for arg in args.arg_clang_args {
        builder.clang_arg(arg);
    }
    if let Some(s) = args.flag_match {
        builder.match_pat(s);
    }
    if let Some(s) = args.flag_remove_prefix {
        builder.remove_prefix(s);
    }
    if args.flag_builtins {
        builder.builtins();
    }
    if let Some(link) = args.flag_link {
        let mut parts = link.split('=');
        let (lib, kind) = match (parts.next(), parts.next()) {
            (Some(lib), None) => (lib, LinkType::Dynamic),
            (Some(kind), Some(lib)) => {
                (lib,
                 match kind {
                    "static" => LinkType::Static,
                    "dynamic" => LinkType::Dynamic,
                    "framework" => LinkType::Framework,
                    _ => {
                        println!("Link type unknown: {}", kind);
                        exit(1);
                    }
                })
            }
            _ => {
                println!("Wrong link format: {}", link);
                exit(1);
            }
        };
        builder.link(lib, kind);
    }
}

fn get_output(o: &str) -> Box<Write> {
    if o == "-" {
        Box::new(io::stdout())
    } else {
        Box::new(File::create(o).expect(&format!("\"{}\" unwritable", o)))
    }
}

pub fn main() {
    env_logger::LogBuilder::new().parse("bindgen=warn").init().unwrap();

    let args: Args = docopt::Docopt::new(USAGE)
                         .and_then(|d| d.decode())
                         .unwrap_or_else(|e| e.exit());
    debug!("{:?}", args);

    let output = get_output(&args.flag_output);

    let mut builder = Builder::new();
    args_to_opts(args, &mut builder);
    debug!("{:?}", builder);

    match builder.generate() {
        Ok(bindings) => {
            match bindings.write(output) {
                Ok(()) => (),
                Err(e) => {
                    error!("Unable to write bindings to file. {}", e);
                    exit(-1);
                }
            }
        }
        Err(()) => exit(-1),
    }
}
<commit_msg>Add `--no-derive-debug`.<commit_after>extern crate bindgen;
#[macro_use]
extern crate log;
extern crate docopt;
#[macro_use]
extern crate rustc_serialize;
extern crate env_logger;

use bindgen::{Builder, LinkType};
use std::io::{self, Write};
use std::fs::File;
use std::process::exit;

const USAGE: &'static str = "
Generate C bindings for Rust.

Usage:
  bindgen [options] <file> [-- <clang-args>...]
  bindgen (-h | --help)

Options:
  <clang-args>                 Options passed directly to clang.
  -h, --help                   Display this help message.
  --link=<library>             Link to a dynamic library, can be provided multiple times.
                               <library> is in the format `[kind=]lib`, where `kind` is
                               one of `static`, `dynamic` or `framework`.
  --output=<output>            Write bindings to <output> (- is stdout).
                               [default: -]
  --match=<name>               Only output bindings for definitions from files
                               whose name contains <name>
                               If multiple -match options are provided, files
                               matching any rule are bound to.
  --builtins                   Output bindings for builtin definitions
                               (for example __builtin_va_list)
  --emit-clang-ast             Output the ast (for debugging purposes)
  --override-enum-type=<type>  Override enum type, type name could be
                                 uchar
                                 schar
                                 ushort
                                 sshort
                                 uint
                                 sint
                                 ulong
                                 slong
                                 ulonglong
                                 slonglong
  --use-core                  Use `core` as a base crate for `Option` and such.
                              See also `--ctypes-prefix`.
  --ctypes-prefix=<prefix>    Use this prefix for all the types in the generated
                              code.
                              [default: std::os::raw]
  --remove-prefix=<prefix>    Prefix to remove from all the symbols, like
                              `libfoo_`. The removal is case-insensitive.
  --no-derive-debug           Disable `derive(Debug)` for all generated types.
";

#[derive(Debug, RustcDecodable)]
struct Args {
    arg_file: String,
    arg_clang_args: Vec<String>,
    flag_link: Option<String>,
    flag_output: String,
    flag_match: Option<String>,
    flag_builtins: bool,
    flag_emit_clang_ast: bool,
    flag_override_enum_type: String,
    flag_ctypes_prefix: String,
    flag_use_core: bool,
    flag_remove_prefix: Option<String>,
    // TODO: allow finer control.
    flag_no_derive_debug: bool,
}

fn args_to_opts(args: Args, builder: &mut Builder) {
    builder.header(args.arg_file)
           .emit_ast(args.flag_emit_clang_ast)
           .ctypes_prefix(args.flag_ctypes_prefix
                              .split("::")
                              .map(String::from)
                              .collect::<Vec<_>>())
           .use_core(args.flag_use_core)
           .derive_debug(!args.flag_no_derive_debug)
           .override_enum_ty(args.flag_override_enum_type);
    for arg in args.arg_clang_args {
        builder.clang_arg(arg);
    }
    if let Some(s) = args.flag_match {
        builder.match_pat(s);
    }
    if let Some(s) = args.flag_remove_prefix {
        builder.remove_prefix(s);
    }
    if args.flag_builtins {
        builder.builtins();
    }
    if let Some(link) = args.flag_link {
        let mut parts = link.split('=');
        let (lib, kind) = match (parts.next(), parts.next()) {
            (Some(lib), None) => (lib, LinkType::Dynamic),
            (Some(kind), Some(lib)) => {
                (lib,
                 match kind {
                    "static" => LinkType::Static,
                    "dynamic" => LinkType::Dynamic,
                    "framework" => LinkType::Framework,
                    _ => {
                        println!("Link type unknown: {}", kind);
                        exit(1);
                    }
                })
            }
            _ => {
                println!("Wrong link format: {}", link);
                exit(1);
            }
        };
        builder.link(lib, kind);
    }
}

fn get_output(o: &str) -> Box<Write> {
    if o == "-" {
        Box::new(io::stdout())
    } else {
        Box::new(File::create(o).expect(&format!("\"{}\" unwritable", o)))
    }
}

pub fn main() {
    env_logger::LogBuilder::new().parse("bindgen=warn").init().unwrap();

    let args: Args = docopt::Docopt::new(USAGE)
                         .and_then(|d| d.decode())
                         .unwrap_or_else(|e| e.exit());
    debug!("{:?}", args);

    let output = get_output(&args.flag_output);

    let mut builder = Builder::new();
    args_to_opts(args, &mut builder);
    debug!("{:?}", builder);

    match builder.generate() {
        Ok(bindings) => {
            match bindings.write(output) {
                Ok(()) => (),
                Err(e) => {
                    error!("Unable to write bindings to file. {}", e);
                    exit(-1);
                }
            }
        }
        Err(()) => exit(-1),
    }
}
<|endoftext|>


<commit_before>extern crate i3ipc;
use i3ipc::I3Connection;
use i3ipc::reply::{Node, NodeType};

fn get_nodes<'a, F>(tree: &'a Node, condition: &F) -> Option<Vec<&'a Node>>
    where F: Fn(&Node) -> bool
{
    for node in &tree.nodes {
        if node.focused {
            let r = vec![node];
            return Some(r);
        }
        if let Some(mut n) = get_nodes(&node, condition) {
            n.push(node);
            return Some(n);
        }
    }
    return None;
}

fn get_current_tab<'a>(nodes: &[&'a Node]) -> &'a Node {
    let i = nodes.iter().position(|&n| {
        match n.nodetype { NodeType::Con => false, _ => true }
    }).unwrap();
    // i is Workspace
    // i - 1 is "Root container" for workspace tabs
    // i - 2 is current tab
    nodes[i - 2]
}

fn focus_child(c: &mut I3Connection) -> bool {
    let r = c.command("focus child").unwrap();
    for o in r.outcomes {
        if o.success == false {
            return false;
        }
    }
    return true;
}

fn superfocus(c: &mut I3Connection, direction: &str) {
    let tree = c.get_tree().unwrap();
    let nodes = get_nodes(
        &tree, &|n: &Node| { return n.focused; }
    ).expect("Can not find focused window. Maybe focused window is floating?");
    let current_tab = get_current_tab(&nodes);
    let focus_tab_msg = format!("[con_id=\"{}\"] focus", current_tab.id);
    c.command(&focus_tab_msg).unwrap();
    let focus_direction_msg = "focus ".to_string() + direction;
    c.command(&focus_direction_msg).unwrap();
    while focus_child(c) {}
}

fn main() {
    let mut connection = I3Connection::connect().unwrap();
    superfocus(&mut connection, "right");
}
<commit_msg>Get direction as argument<commit_after>use std::env;
extern crate i3ipc;
use i3ipc::I3Connection;
use i3ipc::reply::{Node, NodeType};

fn get_nodes<'a, F>(tree: &'a Node, condition: &F) -> Option<Vec<&'a Node>>
    where F: Fn(&Node) -> bool
{
    for node in &tree.nodes {
        if node.focused {
            let r = vec![node];
            return Some(r);
        }
        if let Some(mut n) = get_nodes(&node, condition) {
            n.push(node);
            return Some(n);
        }
    }
    return None;
}

fn get_current_tab<'a>(nodes: &[&'a Node]) -> &'a Node {
    let i = nodes.iter().position(|&n| {
        match n.nodetype { NodeType::Con => false, _ => true }
    }).unwrap();
    // i is Workspace
    // i - 1 is "Root container" for workspace tabs
    // i - 2 is current tab
    nodes[i - 2]
}

fn focus_child(c: &mut I3Connection) -> bool {
    let r = c.command("focus child").unwrap();
    for o in r.outcomes {
        if o.success == false {
            return false;
        }
    }
    return true;
}

fn superfocus(c: &mut I3Connection, direction: &str) {
    let tree = c.get_tree().unwrap();
    let nodes = get_nodes(
        &tree, &|n: &Node| { return n.focused; }
    ).expect("Can not find focused window. Maybe focused window is floating?");
    let current_tab = get_current_tab(&nodes);
    let focus_tab_msg = format!("[con_id=\"{}\"] focus", current_tab.id);
    c.command(&focus_tab_msg).unwrap();
    let focus_direction_msg = "focus ".to_string() + direction;
    c.command(&focus_direction_msg).unwrap();
    while focus_child(c) {}
}

fn main() {
    let direction = env::args().nth(1).expect("Missing direction argument");
    let mut connection = I3Connection::connect().unwrap();
    superfocus(&mut connection, &direction);
}
<|endoftext|>


<commit_before>// error-pattern:yummy
#![feature(box_syntax)]
#![feature(rustc_private)]

#![allow(unknown_lints, missing_docs_in_private_items)]

extern crate clippy_lints;
extern crate getopts;
extern crate rustc;
extern crate rustc_driver;
extern crate rustc_errors;
extern crate rustc_plugin;
extern crate syntax;

use rustc_driver::{driver, CompilerCalls, RustcDefaultCalls, Compilation};
use rustc::session::{config, Session};
use rustc::session::config::{Input, ErrorOutputType};
use std::path::PathBuf;
use std::process::{self, Command};
use syntax::ast;
use std::io::{self, Write};

extern crate cargo_metadata;

struct ClippyCompilerCalls {
    default: RustcDefaultCalls,
    run_lints: bool,
}

impl ClippyCompilerCalls {
    fn new(run_lints: bool) -> Self {
        ClippyCompilerCalls {
            default: RustcDefaultCalls,
            run_lints: run_lints,
        }
    }
}

impl<'a> CompilerCalls<'a> for ClippyCompilerCalls {
    fn early_callback(
        &mut self,
        matches: &getopts::Matches,
        sopts: &config::Options,
        cfg: &ast::CrateConfig,
        descriptions: &rustc_errors::registry::Registry,
        output: ErrorOutputType,
    ) -> Compilation {
        self.default
            .early_callback(matches, sopts, cfg, descriptions, output)
    }
    fn no_input(
        &mut self,
        matches: &getopts::Matches,
        sopts: &config::Options,
        cfg: &ast::CrateConfig,
        odir: &Option<PathBuf>,
        ofile: &Option<PathBuf>,
        descriptions: &rustc_errors::registry::Registry,
    ) -> Option<(Input, Option<PathBuf>)> {
        self.default
            .no_input(matches, sopts, cfg, odir, ofile, descriptions)
    }
    fn late_callback(
        &mut self,
        matches: &getopts::Matches,
        sess: &Session,
        input: &Input,
        odir: &Option<PathBuf>,
        ofile: &Option<PathBuf>,
    ) -> Compilation {
        self.default
            .late_callback(matches, sess, input, odir, ofile)
    }
    fn build_controller(&mut self, sess: &Session, matches: &getopts::Matches) -> driver::CompileController<'a> {
        let mut control = self.default.build_controller(sess, matches);

        if self.run_lints {
            let old = std::mem::replace(&mut control.after_parse.callback, box |_| {});
            control.after_parse.callback = Box::new(move |state| {
                {
                    let mut registry = rustc_plugin::registry::Registry::new(state.session,
                                                                             state
                                                                                .krate
                                                                                .as_ref()
                                                                                .expect("at this compilation stage \
                                                                                the krate must be parsed")
                                                                                .span);
                    registry.args_hidden = Some(Vec::new());
                    clippy_lints::register_plugins(&mut registry);

                    let rustc_plugin::registry::Registry {
                        early_lint_passes,
                        late_lint_passes,
                        lint_groups,
                        llvm_passes,
                        attributes,
                        ..
                    } = registry;
                    let sess = &state.session;
                    let mut ls = sess.lint_store.borrow_mut();
                    for pass in early_lint_passes {
                        ls.register_early_pass(Some(sess), true, pass);
                    }
                    for pass in late_lint_passes {
                        ls.register_late_pass(Some(sess), true, pass);
                    }

                    for (name, to) in lint_groups {
                        ls.register_group(Some(sess), true, name, to);
                    }

                    sess.plugin_llvm_passes.borrow_mut().extend(llvm_passes);
                    sess.plugin_attributes.borrow_mut().extend(attributes);
                }
                old(state);
            });
        }

        control
    }
}

use std::path::Path;

const CARGO_CLIPPY_HELP: &str = r#"Checks a package to catch common mistakes and improve your Rust code.

Usage:
    cargo clippy [options] [--] [<opts>...]

Common options:
    -h, --help               Print this message
    --features               Features to compile for the package
    -V, --version            Print version info and exit

Other options are the same as `cargo rustc`.

To allow or deny a lint from the command line you can use `cargo clippy --`
with:

    -W --warn OPT       Set lint warnings
    -A --allow OPT      Set lint allowed
    -D --deny OPT       Set lint denied
    -F --forbid OPT     Set lint forbidden

The feature `cargo-clippy` is automatically defined for convenience. You can use
it to allow or deny lints from the code, eg.:

    #[cfg_attr(feature = "cargo-clippy", allow(needless_lifetimes))]
"#;

#[allow(print_stdout)]
fn show_help() {
    println!("{}", CARGO_CLIPPY_HELP);
}

#[allow(print_stdout)]
fn show_version() {
    println!("{}", env!("CARGO_PKG_VERSION"));
}

pub fn main() {
    use std::env;

    if env::var("CLIPPY_DOGFOOD").map(|_| true).unwrap_or(false) {
        panic!("yummy");
    }

    // Check for version and help flags even when invoked as 'cargo-clippy'
    if std::env::args().any(|a| a == "--help" || a == "-h") {
        show_help();
        return;
    }
    if std::env::args().any(|a| a == "--version" || a == "-V") {
        show_version();
        return;
    }

    if let Some("clippy") = std::env::args().nth(1).as_ref().map(AsRef::as_ref) {
        // this arm is executed on the initial call to `cargo clippy`

        let manifest_path_arg = std::env::args()
            .skip(2)
            .find(|val| val.starts_with("--manifest-path="));

        let mut metadata =
            if let Ok(metadata) = cargo_metadata::metadata(manifest_path_arg.as_ref().map(AsRef::as_ref)) {
                metadata
            } else {
                let _ = io::stderr().write_fmt(format_args!("error: Could not obtain cargo metadata.\n"));
                process::exit(101);
            };

        let manifest_path = manifest_path_arg.map(|arg| PathBuf::from(Path::new(&arg["--manifest-path=".len()..])));

        let current_dir = std::env::current_dir();

        let package_index = metadata
            .packages
            .iter()
            .position(|package| {
                let package_manifest_path = Path::new(&package.manifest_path);
                if let Some(ref manifest_path) = manifest_path {
                    package_manifest_path == manifest_path
                } else {
                    let current_dir = current_dir
                        .as_ref()
                        .expect("could not read current directory");
                    let package_manifest_directory = package_manifest_path
                        .parent()
                        .expect("could not find parent directory of package manifest");
                    package_manifest_directory == current_dir
                }
            })
            .expect("could not find matching package");
        let package = metadata.packages.remove(package_index);
        for target in package.targets {
            let args = std::env::args().skip(2);
            if let Some(first) = target.kind.get(0) {
                if target.kind.len() > 1 || first.ends_with("lib") {
                    if let Err(code) = process(std::iter::once("--lib".to_owned()).chain(args)) {
                        std::process::exit(code);
                    }
                } else if ["bin", "example", "test", "bench"].contains(&&**first) {
                    if let Err(code) = process(vec![format!("--{}", first), target.name]
                                                   .into_iter()
                                                   .chain(args)) {
                        std::process::exit(code);
                    }
                }
            } else {
                panic!("badly formatted cargo metadata: target::kind is an empty array");
            }
        }
    } else {
        // this arm is executed when cargo-clippy runs `cargo rustc` with the `RUSTC`
        // env var set to itself

        let home = option_env!("RUSTUP_HOME").or(option_env!("MULTIRUST_HOME"));
        let toolchain = option_env!("RUSTUP_TOOLCHAIN").or(option_env!("MULTIRUST_TOOLCHAIN"));
        let sys_root = if let (Some(home), Some(toolchain)) = (home, toolchain) {
            format!("{}/toolchains/{}", home, toolchain)
        } else {
            option_env!("SYSROOT")
                .map(|s| s.to_owned())
                .or_else(|| {
                    Command::new("rustc")
                        .arg("--print")
                        .arg("sysroot")
                        .output()
                        .ok()
                        .and_then(|out| String::from_utf8(out.stdout).ok())
                        .map(|s| s.trim().to_owned())
                })
                .expect("need to specify SYSROOT env var during clippy compilation, or use rustup or multirust")
        };

        rustc_driver::in_rustc_thread(|| {
            // this conditional check for the --sysroot flag is there so users can call
            // `cargo-clippy` directly
            // without having to pass --sysroot or anything
            let mut args: Vec<String> = if env::args().any(|s| s == "--sysroot") {
                env::args().collect()
            } else {
                env::args()
                    .chain(Some("--sysroot".to_owned()))
                    .chain(Some(sys_root))
                    .collect()
            };

            // this check ensures that dependencies are built but not linted and the final
            // crate is
            // linted but not built
            let clippy_enabled = env::args().any(|s| s == "-Zno-trans");

            if clippy_enabled {
                args.extend_from_slice(&["--cfg".to_owned(), r#"feature="cargo-clippy""#.to_owned()]);
            }

            let mut ccc = ClippyCompilerCalls::new(clippy_enabled);
            let (result, _) = rustc_driver::run_compiler(&args, &mut ccc, None, None);
            if let Err(err_count) = result {
                if err_count > 0 {
                    std::process::exit(1);
                }
            }
        })
                .expect("rustc_thread failed");
    }
}

fn process<I>(old_args: I) -> Result<(), i32>
    where I: Iterator<Item = String>
{

    let mut args = vec!["rustc".to_owned()];

    let mut found_dashes = false;
    for arg in old_args {
        found_dashes |= arg == "--";
        args.push(arg);
    }
    if !found_dashes {
        args.push("--".to_owned());
    }
    args.push("-Zno-trans".to_owned());
    args.push("--cfg".to_owned());
    args.push(r#"feature="cargo-clippy""#.to_owned());

    let path = std::env::current_exe().expect("current executable path invalid");
    let exit_status = std::process::Command::new("cargo")
        .args(&args)
        .env("RUSTC", path)
        .spawn()
        .expect("could not run cargo")
        .wait()
        .expect("failed to wait for cargo?");

    if exit_status.success() {
        Ok(())
    } else {
        Err(exit_status.code().unwrap_or(-1))
    }
}
<commit_msg>Allow cargo-clippy to work in subdirectories<commit_after>// error-pattern:yummy
#![feature(box_syntax)]
#![feature(rustc_private)]

#![allow(unknown_lints, missing_docs_in_private_items)]

extern crate clippy_lints;
extern crate getopts;
extern crate rustc;
extern crate rustc_driver;
extern crate rustc_errors;
extern crate rustc_plugin;
extern crate syntax;

use rustc_driver::{driver, CompilerCalls, RustcDefaultCalls, Compilation};
use rustc::session::{config, Session};
use rustc::session::config::{Input, ErrorOutputType};
use std::path::PathBuf;
use std::process::{self, Command};
use syntax::ast;
use std::io::{self, Write};

extern crate cargo_metadata;

struct ClippyCompilerCalls {
    default: RustcDefaultCalls,
    run_lints: bool,
}

impl ClippyCompilerCalls {
    fn new(run_lints: bool) -> Self {
        ClippyCompilerCalls {
            default: RustcDefaultCalls,
            run_lints: run_lints,
        }
    }
}

impl<'a> CompilerCalls<'a> for ClippyCompilerCalls {
    fn early_callback(
        &mut self,
        matches: &getopts::Matches,
        sopts: &config::Options,
        cfg: &ast::CrateConfig,
        descriptions: &rustc_errors::registry::Registry,
        output: ErrorOutputType,
    ) -> Compilation {
        self.default
            .early_callback(matches, sopts, cfg, descriptions, output)
    }
    fn no_input(
        &mut self,
        matches: &getopts::Matches,
        sopts: &config::Options,
        cfg: &ast::CrateConfig,
        odir: &Option<PathBuf>,
        ofile: &Option<PathBuf>,
        descriptions: &rustc_errors::registry::Registry,
    ) -> Option<(Input, Option<PathBuf>)> {
        self.default
            .no_input(matches, sopts, cfg, odir, ofile, descriptions)
    }
    fn late_callback(
        &mut self,
        matches: &getopts::Matches,
        sess: &Session,
        input: &Input,
        odir: &Option<PathBuf>,
        ofile: &Option<PathBuf>,
    ) -> Compilation {
        self.default
            .late_callback(matches, sess, input, odir, ofile)
    }
    fn build_controller(&mut self, sess: &Session, matches: &getopts::Matches) -> driver::CompileController<'a> {
        let mut control = self.default.build_controller(sess, matches);

        if self.run_lints {
            let old = std::mem::replace(&mut control.after_parse.callback, box |_| {});
            control.after_parse.callback = Box::new(move |state| {
                {
                    let mut registry = rustc_plugin::registry::Registry::new(state.session,
                                                                             state
                                                                                .krate
                                                                                .as_ref()
                                                                                .expect("at this compilation stage \
                                                                                the krate must be parsed")
                                                                                .span);
                    registry.args_hidden = Some(Vec::new());
                    clippy_lints::register_plugins(&mut registry);

                    let rustc_plugin::registry::Registry {
                        early_lint_passes,
                        late_lint_passes,
                        lint_groups,
                        llvm_passes,
                        attributes,
                        ..
                    } = registry;
                    let sess = &state.session;
                    let mut ls = sess.lint_store.borrow_mut();
                    for pass in early_lint_passes {
                        ls.register_early_pass(Some(sess), true, pass);
                    }
                    for pass in late_lint_passes {
                        ls.register_late_pass(Some(sess), true, pass);
                    }

                    for (name, to) in lint_groups {
                        ls.register_group(Some(sess), true, name, to);
                    }

                    sess.plugin_llvm_passes.borrow_mut().extend(llvm_passes);
                    sess.plugin_attributes.borrow_mut().extend(attributes);
                }
                old(state);
            });
        }

        control
    }
}

use std::path::Path;

const CARGO_CLIPPY_HELP: &str = r#"Checks a package to catch common mistakes and improve your Rust code.

Usage:
    cargo clippy [options] [--] [<opts>...]

Common options:
    -h, --help               Print this message
    --features               Features to compile for the package
    -V, --version            Print version info and exit

Other options are the same as `cargo rustc`.

To allow or deny a lint from the command line you can use `cargo clippy --`
with:

    -W --warn OPT       Set lint warnings
    -A --allow OPT      Set lint allowed
    -D --deny OPT       Set lint denied
    -F --forbid OPT     Set lint forbidden

The feature `cargo-clippy` is automatically defined for convenience. You can use
it to allow or deny lints from the code, eg.:

    #[cfg_attr(feature = "cargo-clippy", allow(needless_lifetimes))]
"#;

#[allow(print_stdout)]
fn show_help() {
    println!("{}", CARGO_CLIPPY_HELP);
}

#[allow(print_stdout)]
fn show_version() {
    println!("{}", env!("CARGO_PKG_VERSION"));
}

fn has_prefix<'a, T: PartialEq, I: Iterator<Item = &'a T>>(v: &'a [T], itr: I) -> bool {
    v.iter().zip(itr).all(|(a, b)| a == b)
}

pub fn main() {
    use std::env;

    if env::var("CLIPPY_DOGFOOD").map(|_| true).unwrap_or(false) {
        panic!("yummy");
    }

    // Check for version and help flags even when invoked as 'cargo-clippy'
    if std::env::args().any(|a| a == "--help" || a == "-h") {
        show_help();
        return;
    }
    if std::env::args().any(|a| a == "--version" || a == "-V") {
        show_version();
        return;
    }

    if let Some("clippy") = std::env::args().nth(1).as_ref().map(AsRef::as_ref) {
        // this arm is executed on the initial call to `cargo clippy`

        let manifest_path_arg = std::env::args()
            .skip(2)
            .find(|val| val.starts_with("--manifest-path="));

        let mut metadata =
            if let Ok(metadata) = cargo_metadata::metadata(manifest_path_arg.as_ref().map(AsRef::as_ref)) {
                metadata
            } else {
                let _ = io::stderr().write_fmt(format_args!("error: Could not obtain cargo metadata.\n"));
                process::exit(101);
            };

        let manifest_path = manifest_path_arg.map(|arg| PathBuf::from(Path::new(&arg["--manifest-path=".len()..])));

        let package_index = {
                let mut iterator = metadata.packages.iter();

                if let Some(ref manifest_path) = manifest_path {
                    iterator.position(|package| {
                        let package_manifest_path = Path::new(&package.manifest_path);
                        package_manifest_path == manifest_path
                    })
                } else {
                    let current_dir = std::env::current_dir()
                        .expect("could not read current directory")
                        .canonicalize()
                        .expect("current directory cannot be canonicalized");
                    let current_dir_components = current_dir.components().collect::<Vec<_>>();

                    // This gets the most-recent parent (the one that takes the fewest `cd ..`s to
                    // reach).
                    iterator
                        .enumerate()
                        .filter_map(|(i, package)| {
                            let package_manifest_path = Path::new(&package.manifest_path);
                            let canonical_path = package_manifest_path
                                .parent()
                                .expect("could not find parent directory of package manifest")
                                .canonicalize()
                                .expect("package directory cannot be canonicalized");

                            // TODO: We can do this in `O(1)` by combining the `len` and the
                            //       iteration.
                            let components = canonical_path.components().collect::<Vec<_>>();
                            if has_prefix(&current_dir_components, components.iter()) {
                                Some((i, components.len()))
                            } else {
                                None
                            }
                        })
                        .max_by_key(|&(_, length)| length)
                        .map(|(i, _)| i)
                }
            }
            .expect("could not find matching package");

        let package = metadata.packages.remove(package_index);
        for target in package.targets {
            let args = std::env::args().skip(2);
            if let Some(first) = target.kind.get(0) {
                if target.kind.len() > 1 || first.ends_with("lib") {
                    if let Err(code) = process(std::iter::once("--lib".to_owned()).chain(args)) {
                        std::process::exit(code);
                    }
                } else if ["bin", "example", "test", "bench"].contains(&&**first) {
                    if let Err(code) = process(vec![format!("--{}", first), target.name]
                                                   .into_iter()
                                                   .chain(args)) {
                        std::process::exit(code);
                    }
                }
            } else {
                panic!("badly formatted cargo metadata: target::kind is an empty array");
            }
        }
    } else {
        // this arm is executed when cargo-clippy runs `cargo rustc` with the `RUSTC`
        // env var set to itself

        let home = option_env!("RUSTUP_HOME").or(option_env!("MULTIRUST_HOME"));
        let toolchain = option_env!("RUSTUP_TOOLCHAIN").or(option_env!("MULTIRUST_TOOLCHAIN"));
        let sys_root = if let (Some(home), Some(toolchain)) = (home, toolchain) {
            format!("{}/toolchains/{}", home, toolchain)
        } else {
            option_env!("SYSROOT")
                .map(|s| s.to_owned())
                .or_else(|| {
                    Command::new("rustc")
                        .arg("--print")
                        .arg("sysroot")
                        .output()
                        .ok()
                        .and_then(|out| String::from_utf8(out.stdout).ok())
                        .map(|s| s.trim().to_owned())
                })
                .expect("need to specify SYSROOT env var during clippy compilation, or use rustup or multirust")
        };

        rustc_driver::in_rustc_thread(|| {
            // this conditional check for the --sysroot flag is there so users can call
            // `cargo-clippy` directly
            // without having to pass --sysroot or anything
            let mut args: Vec<String> = if env::args().any(|s| s == "--sysroot") {
                env::args().collect()
            } else {
                env::args()
                    .chain(Some("--sysroot".to_owned()))
                    .chain(Some(sys_root))
                    .collect()
            };

            // this check ensures that dependencies are built but not linted and the final
            // crate is
            // linted but not built
            let clippy_enabled = env::args().any(|s| s == "-Zno-trans");

            if clippy_enabled {
                args.extend_from_slice(&["--cfg".to_owned(), r#"feature="cargo-clippy""#.to_owned()]);
            }

            let mut ccc = ClippyCompilerCalls::new(clippy_enabled);
            let (result, _) = rustc_driver::run_compiler(&args, &mut ccc, None, None);
            if let Err(err_count) = result {
                if err_count > 0 {
                    std::process::exit(1);
                }
            }
        })
                .expect("rustc_thread failed");
    }
}

fn process<I>(old_args: I) -> Result<(), i32>
    where I: Iterator<Item = String>
{

    let mut args = vec!["rustc".to_owned()];

    let mut found_dashes = false;
    for arg in old_args {
        found_dashes |= arg == "--";
        args.push(arg);
    }
    if !found_dashes {
        args.push("--".to_owned());
    }
    args.push("-Zno-trans".to_owned());
    args.push("--cfg".to_owned());
    args.push(r#"feature="cargo-clippy""#.to_owned());

    let path = std::env::current_exe().expect("current executable path invalid");
    let exit_status = std::process::Command::new("cargo")
        .args(&args)
        .env("RUSTC", path)
        .spawn()
        .expect("could not run cargo")
        .wait()
        .expect("failed to wait for cargo?");

    if exit_status.success() {
        Ok(())
    } else {
        Err(exit_status.code().unwrap_or(-1))
    }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate clap;
#[macro_use]
extern crate nom;
#[macro_use]
extern crate quick_error;
extern crate walkdir;

use clap::{App, AppSettings};

use std::env;
use std::path::{Path, PathBuf};

mod error;
mod file;
mod parser;
mod questions;
mod stamp;
mod template;
mod template_file;

use error::{Error, Result};
use template_file::RenderedFile;

fn run() -> Result<()> {
    let yaml = load_yaml!("cli.yaml");
    let matches = App::from_yaml(yaml)
        .setting(AppSettings::ArgRequiredElseHelp)
        .get_matches();

    let source_arg = matches
        .value_of("source")
        .expect("Source is a required argument");
    let dest_arg = matches.value_of("destination").unwrap_or("");

    let source = find_source(source_arg)?;
    let dest = find_destination(dest_arg)?;

    let files = file::read_all_files(&source)?;
    let (templates, vars) = stamp::compile_templates(&files);

    let conf = questions::get_vars(&vars)?;
    let rendered = templates
        .iter()
        .map(|t| t.render(&conf))
        .collect::<Result<Vec<RenderedFile>>>()?;

    stamp::write_files(&dest, &rendered)?;

    Ok(())
}

pub fn find_source(source_arg: &str) -> Result<PathBuf> {
    let mut path = Path::new(source_arg).to_owned();
    if path.is_relative() {
        let in_current_dir = env::current_dir()?.join(&path);
        if in_current_dir.exists() {
            return Ok(in_current_dir);
        } else {
            path = env::home_dir()
                .ok_or(Error::HomeDirNotAccessible)?
                .join(".stamps")
                .join(&path);
        }
    }
    if path.exists() {
        Ok(path)
    } else {
        Err(Error::StampNotFound)
    }
}

fn find_destination(dest_arg: &str) -> Result<PathBuf> {
    let path = Path::new(dest_arg).to_owned();
    if path.is_relative() {
        Ok(env::current_dir()?.join(&path))
    } else {
        Ok(path)
    }
}

fn main() {
    run().unwrap()
}
<commit_msg>Refactor<commit_after>#[macro_use]
extern crate clap;
#[macro_use]
extern crate nom;
#[macro_use]
extern crate quick_error;
extern crate walkdir;

mod error;
mod file;
mod parser;
mod questions;
mod stamp;
mod template;
mod template_file;

use clap::{App, AppSettings};

use std::env;
use std::path::{Path, PathBuf};

use error::{Error, Result};
use template_file::RenderedFile;

fn run() -> Result<()> {
    let yaml = load_yaml!("cli.yaml");
    let matches = App::from_yaml(yaml)
        .setting(AppSettings::ArgRequiredElseHelp)
        .get_matches();

    let source_arg = matches
        .value_of("source")
        .expect("Source is a required argument");
    let dest_arg = matches.value_of("destination").unwrap_or("");

    let source = find_source(source_arg)?;
    let dest = find_destination(dest_arg)?;

    let files = file::read_all_files(&source)?;
    let (templates, vars) = stamp::compile_templates(&files);

    let conf = questions::get_vars(&vars)?;
    let rendered = templates
        .iter()
        .map(|t| t.render(&conf))
        .collect::<Result<Vec<RenderedFile>>>()?;

    stamp::write_files(&dest, &rendered)?;

    Ok(())
}

pub fn find_source(source_arg: &str) -> Result<PathBuf> {
    let mut path = Path::new(source_arg).to_owned();
    if path.is_relative() {
        let in_current_dir = env::current_dir()?.join(&path);
        if in_current_dir.exists() {
            return Ok(in_current_dir);
        } else {
            path = env::home_dir()
                .ok_or(Error::HomeDirNotAccessible)?
                .join(".stamps")
                .join(&path);
        }
    }
    if path.exists() {
        Ok(path)
    } else {
        Err(Error::StampNotFound)
    }
}

fn find_destination(dest_arg: &str) -> Result<PathBuf> {
    let path = Path::new(dest_arg).to_owned();
    if path.is_relative() {
        Ok(env::current_dir()?.join(&path))
    } else {
        Ok(path)
    }
}

fn main() {
    run().unwrap()
}
<|endoftext|>


<commit_before>#![feature(path_ext)]
#![feature(convert)]
extern crate mysql;
extern crate toml;
#[macro_use]
extern crate lazy_static;

mod lib;

use mysql::conn::MyOpts;
use std::default::Default;
use mysql::conn::pool;
use mysql::value::from_value;

use toml::Table;
use toml::Value;

use lib::config;
use lib::downloader::DownloadDB;
use lib::downloader::Downloader;

static VERSION : &'static str = "0.1"; // String not valid

lazy_static! {
    static ref CONFIG: Table = {
        println!("Starting yayd-backend v{}",&VERSION);
        config::init_config() //return
    };

}

fn main() {
    let opts = mysql_options();
    let pool = match pool::MyPool::new(opts) {
        Ok(conn) => { println!("Connected successfully."); conn},
        Err(err) => panic!("Unable to establish a connection!\n{}",err),
    };
    loop {
    	if let Some(result) = request_entry(& pool) {
    		if result.playlist {
    			println!("Playlist not supported atm!");
    			//TODO: set playlist entry to err
    		}
    		handle_request(result);
    	} else {
    		println!("Pausing..");
    		std::thread::sleep_ms(5000);
    		println!("End pausing..");
    	}
    }

	// let downloader = downloader::Downloader::new(download_db);
	// downloader.download_video();

    println!("EOL!");
}

fn handle_request(downl_db: DownloadDB){
	let download = Downloader::new(downl_db);
	let name = match download.get_file_name() {
		Ok(v) => v,
		Err(e) => {
			println!("{:?}", e);
			return;
		}
	};
	println!("Filename: {}", name);
}

fn request_entry(pool: & pool::MyPool) -> Option<DownloadDB> {
	let mut conn = pool.get_conn().unwrap();
    let mut stmt = conn.prepare("SELECT queries.qid,url,type,quality FROM querydetails \
                    INNER JOIN queries \
                    ON querydetails.qid = queries.qid \
                    WHERE querydetails.code = 0 \
                    ORDER BY queries.created \
                    LIMIT 1").unwrap();
    let mut result = stmt.execute(&[]).unwrap();
    let result = match result.next() {
    	Some(val) => val.unwrap(),
    	None => {return None; },
    };
    println!("Result: {:?}", result[0]);
    println!("result str: {}", result[1].into_str());
    //url: &str, quality: i16, qid: i64, folderFormat: &str, pool: MyPool+
    let download_db = DownloadDB { url: from_value::<String>(&result[1]),
												quality: from_value::<i16>(&result[3]),
												qid: from_value::<i64>(&result[0]),
												folder_format: "/home/dev".into(),
												pool: pool.clone(),
												download_limit: 6,
												playlist: false, //TEMP
												compress: false };
	Some(download_db)
}

///Set options for the connection
fn mysql_options() -> MyOpts {
    let dbconfig = CONFIG.get("db").unwrap().clone();
    let dbconfig = dbconfig.as_table().unwrap(); // shadow binding to workaround borrow / lifetime problems

    MyOpts {
        //tcp_addr: Some(dbconfig.get("ip").unwrap().as_str().clone()),
        tcp_addr: get_option_string(dbconfig,"ip"),
        tcp_port: dbconfig.get("port").unwrap().as_integer().unwrap() as u16,
        //TODO: value does support Encodable -> set as encodable..
        user: get_option_string(dbconfig,"user"),
        pass: get_option_string(dbconfig, "password"),
        db_name: get_option_string(dbconfig, "db"),
        ..Default::default() // set other to default
    }
}

///Converts a toml::Value to a Option<String> for mysql::MyOpts
fn get_option_string(table: & Table,key: & str) -> Option<String> {
    let val: Value = table.get(key).unwrap().clone();
    if let toml::Value::String(s) = val {
        Some(s)
    } else { unreachable!() }
}

// fn get_option_int(table: & Table, key: & str) -> Option<int> {
//     let val: Value = table.get(key).unwrap().clone();
//     if let toml::Value::
// }<commit_msg>*more error handling<commit_after>#![feature(path_ext)]
#![feature(convert)]
extern crate mysql;
extern crate toml;
#[macro_use]
extern crate lazy_static;

mod lib;

use mysql::conn::MyOpts;
use std::default::Default;
use mysql::conn::pool;
use mysql::value::from_value;

use toml::Table;
use toml::Value;

use lib::config;
use lib::downloader::DownloadDB;
use lib::downloader::Downloader;
use lib::downloader::DownloadError;

static VERSION : &'static str = "0.1"; // String not valid

lazy_static! {
    static ref CONFIG: Table = {
        println!("Starting yayd-backend v{}",&VERSION);
        config::init_config() //return
    };

}

fn main() {
    let opts = mysql_options();
    let pool = match pool::MyPool::new(opts) {
        Ok(conn) => { println!("Connected successfully."); conn},
        Err(err) => panic!("Unable to establish a connection!\n{}",err),
    };
    loop {
    	if let Some(result) = request_entry(& pool) {
    		if result.playlist {
    			println!("Playlist not supported atm!");
    			//TODO: set playlist entry to err
    		}
    		handle_request(result);
    	} else {
    		println!("Pausing..");
    		std::thread::sleep_ms(5000);
    		println!("End pausing..");
    	}
    }

	// let downloader = downloader::Downloader::new(download_db);
	// downloader.download_video();

    println!("EOL!");
}

fn handle_request(downl_db: DownloadDB){
	let download = Downloader::new(downl_db);
	let name = match download.get_file_name() {
		Ok(v) => v,
		Err(DownloadError::DMCAError) => {
			println!("DMCA error!");
			return;
		},
		Err(e) => {
			println!("Unknown error: {:?}", e);
			return;
		}
	};
	println!("Filename: {}", name);
}

fn request_entry(pool: & pool::MyPool) -> Option<DownloadDB> {
	let mut conn = pool.get_conn().unwrap();
    let mut stmt = conn.prepare("SELECT queries.qid,url,type,quality FROM querydetails \
                    INNER JOIN queries \
                    ON querydetails.qid = queries.qid \
                    WHERE querydetails.code = 0 \
                    ORDER BY queries.created \
                    LIMIT 1").unwrap();
    let mut result = stmt.execute(&[]).unwrap();
    let result = match result.next() {
    	Some(val) => val.unwrap(),
    	None => {return None; },
    };
    println!("Result: {:?}", result[0]);
    println!("result str: {}", result[1].into_str());
    //url: &str, quality: i16, qid: i64, folderFormat: &str, pool: MyPool+
    let download_db = DownloadDB { url: from_value::<String>(&result[1]),
												quality: from_value::<i16>(&result[3]),
												qid: from_value::<i64>(&result[0]),
												folder_format: "/home/dev".into(),
												pool: pool.clone(),
												download_limit: 6,
												playlist: false, //TEMP
												compress: false };
	Some(download_db)
}

///Set options for the connection
fn mysql_options() -> MyOpts {
    let dbconfig = CONFIG.get("db").unwrap().clone();
    let dbconfig = dbconfig.as_table().unwrap(); // shadow binding to workaround borrow / lifetime problems

    MyOpts {
        //tcp_addr: Some(dbconfig.get("ip").unwrap().as_str().clone()),
        tcp_addr: get_option_string(dbconfig,"ip"),
        tcp_port: dbconfig.get("port").unwrap().as_integer().unwrap() as u16,
        //TODO: value does support Encodable -> set as encodable..
        user: get_option_string(dbconfig,"user"),
        pass: get_option_string(dbconfig, "password"),
        db_name: get_option_string(dbconfig, "db"),
        ..Default::default() // set other to default
    }
}

///Converts a toml::Value to a Option<String> for mysql::MyOpts
fn get_option_string(table: & Table,key: & str) -> Option<String> {
    let val: Value = table.get(key).unwrap().clone();
    if let toml::Value::String(s) = val {
        Some(s)
    } else { unreachable!() }
}

// fn get_option_int(table: & Table, key: & str) -> Option<int> {
//     let val: Value = table.get(key).unwrap().clone();
//     if let toml::Value::
// }<|endoftext|>


<commit_before>// rust version of breakfast tool
// BreakFast is a toolkit for detecting chromosomal rearrangements
// based on RNA-seq data.

use std::env;
use std::mem;
use std::path::Path;
use std::process::Command;

#[macro_use]
extern crate clap;
#[macro_use]
extern crate shells;
#[macro_use]
extern crate bio;
#[macro_use]
extern crate regex;

use regex::Regex;

mod cli;
mod lib;
// struct for having breakfast options
struct BfOptions {
    anchor_len :    i32,
    max_frag_len:   i32,
    min_mapq:       i32,
    orientation:    &'static str,
    all_reads:      bool,
    discard_duplicates: &'static str,
    min_reads:      &'static str,
    freq_above:     i32,
}

fn main() {
  let defaults = BfOptions{ anchor_len : 0,
                            max_frag_len: 5000,
                            min_mapq: 15,
                            orientation:  "fr",
                            all_reads:  false,
                            discard_duplicates: "both-ends",
                            min_reads:    "0-0-0",
                            freq_above: 0};

  let mut anchor_len: i32   = defaults.anchor_len;
  let mut max_frag_len: i32 = defaults.max_frag_len;
  let mut min_mapq: i32     = defaults.min_mapq;
  let mut orientation       = String::from(defaults.orientation);
  let mut all_reads: bool   = defaults.all_reads;
  let mut discard_duplicates = String::from(defaults.discard_duplicates);
  let mut min_reads     = String::from(defaults.min_reads);
  let mut freq_above: i32 = defaults.freq_above;

  //arguments from src/cli.rs
  let matches = cli::build_cli().get_matches();
  /*
  if matches.is_present("anchor-len") {
      anchor_len = matches.value_of("anchor-len").unwrap().parse::<i32>().unwrap();
  }

  if matches.is_present("max-frag-len") {
      max_frag_len = matches.value_of("max-frag-len").unwrap().parse::<i32>().unwrap();
  }

  if matches.is_present("min-mapq") {
      min_mapq = matches.value_of("min-mapq").unwrap().parse::<i32>().unwrap();
  }

  if matches.is_present("all-reads") {
      all_reads = true;
  }

  if matches.is_present("discard-duplicates") {
      discard_duplicates = String::from(matches.value_of("discard-duplicates").unwrap());
  }

  if matches.is_present("min-reads") {
      min_reads = String::from(matches.value_of("min-reads").unwrap());
  }

  if matches.is_present("freq-above") {
      freq_above = matches.value_of("freq-above").unwrap().parse::<i32>().unwrap();
  } */

  if let ("detect", Some(detect)) = matches.subcommand() {
      let bam_file = String::from(detect.value_of("bam_file").unwrap());
      let genome   = String::from(detect.value_of("genome").unwrap());
      let out_prefix  = String::from(detect.value_of("out_prefix").unwrap());

      if detect.is_present("orientation") {
          orientation = String::from(detect.value_of("orientation").unwrap());
      }
      detect_discordant_pairs(bam_file, out_prefix, max_frag_len, min_mapq, orientation);
  }

  if let ("detectspecific", Some(detectspecific)) = matches.subcommand() {
      let bam_file = String::from(detectspecific.value_of("bam_file").unwrap());
      let genome   = String::from(detectspecific.value_of("genome").unwrap());
      let out_prefix  = String::from(detectspecific.value_of("out_prefix").unwrap());
      let donors  = String::from(detectspecific.value_of("donors").unwrap());
      let acceptors  = String::from(detectspecific.value_of("acceptors").unwrap());
  }

  println!("Hello, world!");
}

// DETECT //
//======= //

fn detect_discordant_pairs(sam_path: String, out_prefix: String, max_frag_len: i32,
	min_mapq: i32, orientation: String) {
  println!("{:?}\t{:?}\t{:?}\t{:?}\t{:?}", sam_path, out_prefix, max_frag_len, min_mapq, orientation);

  let mut N = 0;
  let tmp = out_prefix.clone();
  let given_path = Path::new(&tmp);
  let mut sort_tmp_dir = Path::new("./");
  if given_path.is_absolute()  {
        sort_tmp_dir = given_path.parent().unwrap();
  }

  let extn = String::from(".discordant_pairs.tsv.gz");
  let out  = out_prefix + &extn;

  lib::mkdir(&out);
  println!("Searching for discordant read pairs ...");

  let (code, stdout, stderr) = sh!("sam discordant pairs --min-mapq={} {} {} | sort -k1,1 -T {:?}", min_mapq, sam_path, max_frag_len, sort_tmp_dir);

  let mut prev : Vec<&str> = Vec::new();
  for line in stdout.lines() {
      let mut cols: Vec<&str> = line.split('\t').collect();
      if cols.len() < 9 { continue; }

      let r1 = Regex::new(r"N").unwrap();
      let r2 = Regex::new(r"S").unwrap();

      if r1.is_match(cols[5]) || r2.is_match(cols[5]) { continue; }
      else if cols[0].ends_with("/1") || cols[0].ends_with("/2") {
          let tmp : Vec<&str> = cols[0].split('/').collect();
          cols[0] = tmp[0];
      }
      if prev.len() <= 0 { prev = cols.clone(); }
      else if prev.len() > 0 && cols[0] != prev[0] { prev = cols.clone(); }

      let flags = &cols[1].parse::<i32>().unwrap();
      let mut chr   = String::from(cols[2]);
      let mut mchr  = String::from(prev[2]);
      let mut strand = '-';
      let mut mstrand = '-';
      let mut pos  = cols[3].parse::<i32>().unwrap();
      let mut mpos = prev[3].parse::<i32>().unwrap();
      let mut rlen = cols[9].clone().len() as i32;
      let mut mrlen= prev[9].clone().len() as i32;

      if chr.starts_with("chr") {  chr = chr; }
      else { let prefix = String::from("chr"); chr = prefix + &chr;}

      if mchr.starts_with("chr") {  chr = chr ;}
      else { let prefix = String::from("chr"); mchr = prefix + &mchr;}

      if chr.contains("chrM") || mchr.contains("chrM") { continue;}

      if orientation == "fr" {
        if chr > mchr || (chr == mchr && pos > mpos) {
          mem::swap(&mut chr, &mut mchr);
          mem::swap(&mut pos, &mut mpos);
          mem::swap(&mut rlen, &mut mrlen);
          mem::swap(&mut strand, &mut mstrand);
        }
        if mstrand == '+' { mstrand = '-';}
        else { mstrand = '+'; }
      }

      else if orientation == "rf" {
        if chr > mchr || (chr == mchr && pos > mpos) {
          mem::swap(&mut chr, &mut mchr);
          mem::swap(&mut pos, &mut mpos);
          mem::swap(&mut rlen, &mut mrlen);
          mem::swap(&mut strand, &mut mstrand);
        }
        if strand == '+' { strand = '-';}
        else { strand = '+'; }
      }

      else if orientation == "ff" {
        if chr > mchr || (chr == mchr && pos > mpos) {
          mem::swap(&mut chr, &mut mchr);
          mem::swap(&mut pos, &mut mpos);
          mem::swap(&mut rlen, &mut mrlen);

          if mstrand == '-' { mstrand = '+';} else { mstrand = '-'; }
          if strand  == '-' { strand  = '+';} else { strand  = '-'; }
          mem::swap(&mut strand, &mut mstrand);
        }
      } else { panic!("Unsupported read orientation detected."); }

      if strand  == '-' { pos  +=  rlen - 1;}
      if mstrand == '-' { mpos += mrlen - 1;}

      println!("{}\t{}\t{}\t{}\t{}\t{}", chr, strand, pos, mchr, mstrand, mpos);
      N += 1;
    }
    println!("Found {:?} discordant mate pairs.", N);
}
<commit_msg>fn detect_discordant_pairs complete/ fn detect_discordant_reads open<commit_after>// rust version of breakfast tool
// BreakFast is a toolkit for detecting chromosomal rearrangements
// based on RNA-seq data.

use std::env;
use std::mem;
use std::path::Path;
use std::io::prelude::*;
use std::fs;
use std::fs::File;
use std::process::Command;

#[macro_use]
extern crate clap;
#[macro_use]
extern crate shells;
#[macro_use]
extern crate bio;
#[macro_use]
extern crate regex;

use regex::Regex;

mod cli;
mod lib;
// struct for having breakfast options
struct BfOptions {
    anchor_len :    i32,
    max_frag_len:   i32,
    min_mapq:       i32,
    orientation:    &'static str,
    all_reads:      bool,
    discard_duplicates: &'static str,
    min_reads:      &'static str,
    freq_above:     i32,
}

fn main() {
  let defaults = BfOptions{ anchor_len : 0,
                            max_frag_len: 5000,
                            min_mapq: 15,
                            orientation:  "fr",
                            all_reads:  false,
                            discard_duplicates: "both-ends",
                            min_reads:    "0-0-0",
                            freq_above: 0};

  let mut anchor_len: i32   = defaults.anchor_len;
  let mut max_frag_len: i32 = defaults.max_frag_len;
  let mut min_mapq: i32     = defaults.min_mapq;
  let mut orientation       = String::from(defaults.orientation);
  let mut all_reads: bool   = defaults.all_reads;
  let mut discard_duplicates = String::from(defaults.discard_duplicates);
  let mut min_reads     = String::from(defaults.min_reads);
  let mut freq_above: i32 = defaults.freq_above;

  //arguments from src/cli.rs
  let matches = cli::build_cli().get_matches();
  /*
  if matches.is_present("anchor-len") {
      anchor_len = matches.value_of("anchor-len").unwrap().parse::<i32>().unwrap();
  }

  if matches.is_present("max-frag-len") {
      max_frag_len = matches.value_of("max-frag-len").unwrap().parse::<i32>().unwrap();
  }

  if matches.is_present("min-mapq") {
      min_mapq = matches.value_of("min-mapq").unwrap().parse::<i32>().unwrap();
  }

  if matches.is_present("all-reads") {
      all_reads = true;
  }

  if matches.is_present("discard-duplicates") {
      discard_duplicates = String::from(matches.value_of("discard-duplicates").unwrap());
  }

  if matches.is_present("min-reads") {
      min_reads = String::from(matches.value_of("min-reads").unwrap());
  }

  if matches.is_present("freq-above") {
      freq_above = matches.value_of("freq-above").unwrap().parse::<i32>().unwrap();
  } */

  if let ("detect", Some(detect)) = matches.subcommand() {
      let bam_file = String::from(detect.value_of("bam_file").unwrap());
      let genome   = String::from(detect.value_of("genome").unwrap());
      let out_prefix  = String::from(detect.value_of("out_prefix").unwrap());

      if detect.is_present("anchor-len") {
          anchor_len = detect.value_of("anchor-len").unwrap().parse::<i32>().unwrap();
      }

      if detect.is_present("orientation") {
          orientation = String::from(detect.value_of("orientation").unwrap());
      }
      //detect_discordant_pairs(bam_file, out_prefix, max_frag_len, min_mapq, orientation);
      detect_discordant_reads(bam_file, genome, out_prefix, anchor_len);
  }

  if let ("detectspecific", Some(detectspecific)) = matches.subcommand() {
      let bam_file = String::from(detectspecific.value_of("bam_file").unwrap());
      let genome   = String::from(detectspecific.value_of("genome").unwrap());
      let out_prefix  = String::from(detectspecific.value_of("out_prefix").unwrap());
      let donors  = String::from(detectspecific.value_of("donors").unwrap());
      let acceptors  = String::from(detectspecific.value_of("acceptors").unwrap());
  }

  println!("Hello, world!");
}

// DETECT //
//======= //

fn detect_discordant_pairs(sam_path: String, out_prefix: String, max_frag_len: i32,
	min_mapq: i32, orientation: String) {
  println!("{:?}\t{:?}\t{:?}\t{:?}\t{:?}", sam_path, out_prefix, max_frag_len, min_mapq, orientation);

  let mut N = 0;
  let tmp = out_prefix.clone();
  let given_path = Path::new(&tmp);
  let mut sort_tmp_dir = Path::new("./");
  if given_path.is_absolute()  {
        sort_tmp_dir = given_path.parent().unwrap();
  }

  let extn = String::from(".discordant_pairs.tsv.gz");
  let out  = out_prefix + &extn;
  let mut file = File::create("test.txt").unwrap();

  lib::mkdir(&tmp);
  println!("Searching for discordant read pairs ...");

  let (code, stdout, stderr) = sh!("sam discordant pairs --min-mapq={} {} {} | sort -k1,1 -T {:?}", min_mapq, sam_path, max_frag_len, sort_tmp_dir);

  let mut prev : Vec<&str> = Vec::new();
  for line in stdout.lines() {
      let mut cols: Vec<&str> = line.split('\t').collect();
      if cols.len() < 9 { continue; }

      let r1 = Regex::new(r"N").unwrap();
      let r2 = Regex::new(r"S").unwrap();

      if r1.is_match(cols[5]) || r2.is_match(cols[5]) { continue; }
      else if cols[0].ends_with("/1") || cols[0].ends_with("/2") {
          let tmp : Vec<&str> = cols[0].split('/').collect();
          cols[0] = tmp[0];
      }
      if prev.len() <= 0 { prev = cols.clone(); }
      else if prev.len() > 0 && cols[0] != prev[0] { prev = cols.clone(); }

      let flags = &cols[1].parse::<i32>().unwrap();
      let mut chr   = String::from(cols[2]);
      let mut mchr  = String::from(prev[2]);
      let mut strand = '-';
      let mut mstrand = '-';
      let mut pos  = cols[3].parse::<i32>().unwrap();
      let mut mpos = prev[3].parse::<i32>().unwrap();
      let mut rlen = cols[9].clone().len() as i32;
      let mut mrlen= prev[9].clone().len() as i32;

      if chr.starts_with("chr") {  chr = chr; }
      else { let prefix = String::from("chr"); chr = prefix + &chr;}

      if mchr.starts_with("chr") {  chr = chr ;}
      else { let prefix = String::from("chr"); mchr = prefix + &mchr;}

      if chr.contains("chrM") || mchr.contains("chrM") { continue;}

      if orientation == "fr" {
        if chr > mchr || (chr == mchr && pos > mpos) {
          mem::swap(&mut chr, &mut mchr);
          mem::swap(&mut pos, &mut mpos);
          mem::swap(&mut rlen, &mut mrlen);
          mem::swap(&mut strand, &mut mstrand);
        }
        if mstrand == '+' { mstrand = '-';}
        else { mstrand = '+'; }
      }

      else if orientation == "rf" {
        if chr > mchr || (chr == mchr && pos > mpos) {
          mem::swap(&mut chr, &mut mchr);
          mem::swap(&mut pos, &mut mpos);
          mem::swap(&mut rlen, &mut mrlen);
          mem::swap(&mut strand, &mut mstrand);
        }
        if strand == '+' { strand = '-';}
        else { strand = '+'; }
      }

      else if orientation == "ff" {
        if chr > mchr || (chr == mchr && pos > mpos) {
          mem::swap(&mut chr, &mut mchr);
          mem::swap(&mut pos, &mut mpos);
          mem::swap(&mut rlen, &mut mrlen);

          if mstrand == '-' { mstrand = '+';} else { mstrand = '-'; }
          if strand  == '-' { strand  = '+';} else { strand  = '-'; }
          mem::swap(&mut strand, &mut mstrand);
        }
      } else { panic!("Unsupported read orientation detected."); }

      if strand  == '-' { pos  +=  rlen - 1;}
      if mstrand == '-' { mpos += mrlen - 1;}

      let mut out = String::new();
      out.push_str(&chr);out.push_str(&"\t");
      out.push_str(&strand.to_string());out.push_str(&"\t");
      out.push_str(&pos.to_string());out.push_str(&"\t");
      out.push_str(&mchr);out.push_str(&"\t");
      out.push_str(&mstrand.to_string());out.push_str(&"\t");
      out.push_str(&mpos.to_string());out.push_str(&"\t-\n");
      file.write_all(out.as_bytes());
      N += 1;
    }
    let (wcode, wstdout, wstderr) = sh!("gzip -c {:?}.txt > {}", given_path, out);
    let mut rm_tmp = tmp.to_string() + ".txt";
    fs::remove_file(rm_tmp);
    println!("Found {:?} discordant mate pairs.", N);
}

fn detect_discordant_reads(sam_path: String, genome_path: String, out_prefix: String, anchor_len: i32) {

  let mut N: i32 = 0;
  println!("Splitting unaligned reads into {} bp anchors and aligning against the genome...", anchor_len);

  println!("{:?}\t{:?}\t{:?}\t", &sam_path, &anchor_len, &genome_path);

  let (samcode, samout, samerr) = sh!("samtools fasta -f 0x4 {} | fasta split interleaved - {}",  sam_path, anchor_len);

  let (bowtiecode, bowtieout, bowtierr) = sh!("bowtie -f -p1 -v0 -m1 -B1 --suppress 5,6,7,8 {} {}", genome_path, samout);

  for line in bowtieout.lines(){
      println!("{:?}", line);
  }
  println!("SAMCODE {:?}", samcode);
  println!("Error reported code by SAM \t\t\t{:?}", samerr);

  println!("BOWTIECODE {:?}", bowtiecode);
  println!("Error reported code by BOWTIE \t\t\t{:?}", bowtierr);
}
<|endoftext|>