<commit_before>extern crate num;
extern crate portaudio;
extern crate rtlsdr;

use portaudio as pa;
use rtlsdr::{RTLSDRDevice, RTLSDRError};
use num::{Num, Zero};
use num::complex::Complex;
use std::env;
use std::f32::consts::PI;
use std::error::Error;
use pa::error::Error as PAError;
use std::thread;
use std::sync::mpsc::channel;

// Make sure that the buffer size is radix-2, otherwise the read_sync function
// will fail with an error code of -8. I want the buffer sizes between the SDR
// and the audio sizes to match, but since we get two samples per read the
// buffer size needs to be double.
const SDR_BUF_SIZE: usize = 262_144;

// The buffer size for the audio sink.
const AUDIO_BUF_SIZE: usize = 524_288;

// Most other sample rates fail, but this one works for my particular device.
// I will investigate exactly what's happening here and generate a list of
// possible sample rates (if there are others besides this one).
const SDR_SAMPLE_RATE: u32 = 2_400_000;

// Here is the sample rate of the output waveform we'll try to use.
const AUDIO_SAMPLE_RATE: f32 = 48000.0;

// The number of channels being used for audio.
const CHANNELS: i32 = 2;

// We set the gain manually right now since it seems to be a better option
// than using the AGC (the SDR gets blazing hot with the AGC on).
const SDR_GAIN: i32 = 496;

fn main() {
    match run() {
        Ok(()) => (),
        Err(e) => println!("ERROR: {}", e),
    }
}

struct RTLSDR {
    rtlsdr: RTLSDRDevice,
}

unsafe impl Send for RTLSDR {}

fn run() -> Result<(), Box<Error>> {
    let sdr_index: i32 = env::args()
        .nth(1)
        .expect("Please specify the SDR index.")
        .parse::<i32>()?;
    let fm_freq_mhz: f32 = env::args()
        .nth(2)
        .expect("Please specify a center frequency in MHz.")
        .parse::<f32>()?;
    let bandwidth: u32 = env::args()
        .nth(3)
        .expect("Please specify the bandwidth.")
        .parse::<u32>()?;

    // Set up the threading channels for communication between our two threads.
    let (dongle_tx, dongle_rx) = channel();
    let (demod_tx, demod_rx) = channel();

    // This thread only collects data off of the SDR as quickly as possible and
    // sends it out.
    thread::spawn(move || {
        // Initialize the SDR based on the user's input parameters.
        let fm_freq: u32 = (fm_freq_mhz * 1e6) as u32;
        let mut sdr =
            RTLSDR { rtlsdr: init_sdr(sdr_index, fm_freq, bandwidth).unwrap() };

        loop {
            let bytes = sdr.rtlsdr.read_sync(SDR_BUF_SIZE).unwrap();
            let iq_vec = samps_to_cmplx(bytes.as_slice()).unwrap();
            dongle_tx.send(iq_vec).unwrap();
        }
    });

    // This thread does all of the signal processing required to demodulate
    // the signal and get it to a nice audio stream.
    thread::spawn(move || {
        // Decimating down to 100k samples seems to work the best. I don't
        // really quite get why that is, so I'll need to do more research.
        let dec_rate_filt = 6;
        let dec_rate_audio = 4;

        let taps_filt = windowed_sinc(
            bandwidth as f32,
            SDR_SAMPLE_RATE as f32,
            128,
            &hamming,
        );
        let taps_audio = windowed_sinc(
            bandwidth as f32 / dec_rate_filt as f32,
            SDR_SAMPLE_RATE as f32 / dec_rate_filt as f32,
            128,
            &hamming,
        );

        let mut prev = Complex::zero();
        loop {
            // First, low_pass and decimate the raw IQ signal.
            let mut iq_vec = dongle_rx.recv().unwrap();
            iq_vec = filter(&iq_vec, &taps_filt);
            iq_vec = decimate(&iq_vec, dec_rate_filt);

            // Next, demodulate the signal and filter the signal again. The
            // second filter seems to help with the noise after demodulation.
            let res = demod_fm(&iq_vec, prev);
            let mut demod_iq = filter_real(&res.0, &taps_audio);
            demod_iq = decimate(&demod_iq, dec_rate_audio);
            prev = res.1;
            demod_tx.send(demod_iq).unwrap();
        }
    });

    // Initialize the PortAudio class. We use a blocking stream since we're
    // doing processing on the signal and can't keep up with non-blocking
    // timing requirements.
    let audio = try!(pa::PortAudio::new());
    let settings = try!(audio.default_output_stream_settings(
        CHANNELS,
        AUDIO_SAMPLE_RATE as f64,
        AUDIO_BUF_SIZE as u32,
    ));
    let mut stream = try!(audio.open_blocking_stream(settings));
    try!(stream.start());

    loop {
        let demod_iq = demod_rx.recv().unwrap();
        let buffer_frames = (demod_iq.len() / CHANNELS as usize) as u32;
        let mut out_frames = 0;

        while out_frames < buffer_frames {
            out_frames = match stream.write_available() {
                Ok(available) => {
                    match available {
                        pa::StreamAvailable::Frames(frames) => frames as u32,
                        pa::StreamAvailable::InputOverflowed => {
                            return Err(Box::new(PAError::InputOverflowed))
                        }
                        pa::StreamAvailable::OutputUnderflowed => {
                            return Err(Box::new(PAError::OutputUnderflowed))
                        }
                    }
                }
                Err(e) => return Err(Box::new(e)),
            };
        }

        let n_write_samples = buffer_frames as usize * CHANNELS as usize;

        stream.write(
            buffer_frames,
            |output| for ix in 0..n_write_samples {
                output[ix] = demod_iq[ix];
            },
        )?;
    }
}

fn init_sdr(
    sdr_index: i32,
    fm_freq: u32,
    bandwidth: u32,
) -> Result<RTLSDRDevice, RTLSDRError> {
    let mut sdr = try!(rtlsdr::open(sdr_index));
    try!(sdr.set_center_freq(fm_freq));
    try!(sdr.set_sample_rate(SDR_SAMPLE_RATE));
    try!(sdr.set_tuner_bandwidth(bandwidth));
    try!(sdr.set_tuner_gain(SDR_GAIN));
    try!(sdr.set_agc_mode(false));
    try!(sdr.reset_buffer());
    Ok(sdr)
}

/// Decimates a signal by an integer factor.
///
/// # Arguments
///
/// * `signal` - The signal to decimate
/// * `dec_rate` - The decimation factor
///
fn decimate<T: Copy>(signal: &[T], dec_rate: usize) -> Vec<T> {
    let mut ix = 0;
    let new_size = (signal.len() / dec_rate + 1) as usize;
    let mut signal_dec = Vec::<T>::with_capacity(new_size);
    while ix < signal.len() {
        signal_dec.push(signal[ix]);
        ix += dec_rate;
    }
    signal_dec
}

/// Converts samples from the SDR into complex IQ pairs.
///
/// # Arguments
///
/// * `bytes` - The unsigned bytes from the SDR.
///
fn samps_to_cmplx(bytes: &[u8]) -> Option<Vec<Complex<f32>>> {
    // First, check that we've been given an even number of bytes. Not sure
    // what to do if we don't get I and Q.
    let bytes_len = bytes.len();
    if bytes_len % 2 != 0 {
        println!("Received an odd number of samples.");
        return None;

    }

    // Write the values to the complex value and normalize from [0, 255] to
    // [-1, 1].
    let mut iq_vec: Vec<Complex<f32>> = Vec::with_capacity(bytes_len / 2);
    for iq in bytes.chunks(2) {
        let iq_cmplx = Complex::new(
            (iq[0] as f32 - 127.0) / 127.0,
            (iq[1] as f32 - 127.0) / 127.0,
        );
        iq_vec.push(iq_cmplx);
    }
    Some(iq_vec)
}

/// Performs FM demodulation on a complex signal.
///
/// # Arguments
///
/// * `iq` - The IQ samples to demodulate
/// * `prev` - The previous sample from the previous demodulation.
///
fn demod_fm(
    iq: &[Complex<f32>],
    prev: Complex<f32>,
) -> (Vec<f32>, Complex<f32>) {

    let mut p = prev;
    let mut demod_queue: Vec<f32> = Vec::with_capacity(iq.len());

    let gain = SDR_SAMPLE_RATE as f32 / (2.0 * PI * 75e3);

    for samp in iq {
        let conj = p.conj() * samp;
        demod_queue.push(conj.arg() * gain);
        p = *samp;
    }
    (demod_queue, p)
}

/// Filters a Complex vector and returns a new vector.
///
/// # Arguments
///
/// * `samples` - The complex samples to filter
/// * `taps` - The taps of the filter
///
fn filter<T: Copy + Num + Zero>(
    samples: &[Complex<T>],
    taps: &[T],
) -> Vec<Complex<T>> {
    // We'll lose taps.len() - 1 samples from the filtering.
    let mut filt_samps: Vec<Complex<T>> =
        Vec::with_capacity(samples.len() - taps.len() + 1);

    for window in samples.windows(taps.len()) {
        let iter = window.iter().zip(taps.iter());
        let filt_samp = iter.fold(Complex::zero(), |acc, (x, y)| acc + *x * *y);
        filt_samps.push(filt_samp);
    }
    filt_samps
}

/// Filters a real vector and returns a new vector.
///
/// # Arguments
///
/// * `samples` - The real samples to filter
/// * `taps` - The taps of the filter
///
fn filter_real<T: Copy + Num + Zero>(
    samples: &[T],
    taps: &[T],
) -> Vec<T> {
    // We'll lose taps.len() - 1 samples from the filtering.
    let mut filt_samps: Vec<T> =
        Vec::with_capacity(samples.len() - taps.len() + 1);
    for window in samples.windows(taps.len()) {
        let iter = window.iter().zip(taps.iter());
        let filt_samp = iter.fold(T::zero(), |acc, (x, y)| acc + *x * *y);
        filt_samps.push(filt_samp);
    }
    filt_samps
}

/// Generates a vector containing the taps for a Hamming window.
///
/// # Arguments
///
/// * `ntaps` - The number of taps to generate for the filter
///
fn hamming(ntaps: usize) -> Vec<f32> {
    let m: f32 = ntaps as f32 - 1.0;
    let mut taps: Vec<f32> = Vec::with_capacity(ntaps);
    for ix in 0..ntaps {
        taps.push(0.54 - 0.46 * (2.0 * PI * ix as f32 / m).cos());
    }
    taps
}

/// Generates filter taps for a sinc filter with a windowing function.
///
/// # Arguments
///
/// * `cutoff_freq` - The cutoff frequency in Hertz
/// * `sample_rate` - The sample frequency in Hertz
/// * `ntaps` - The number of taps to generate for the window
/// * `window` - The windowing function used to generate the taps for the window
///
fn windowed_sinc(
    cutoff_freq: f32,
    sample_rate: f32,
    ntaps: usize,
    window: &Fn(usize) -> Vec<f32>,
) -> Vec<f32> {
    // Convert the frequency to radians and make it a ratio of the sample rate.
    let wc: f32 = 2.0 * PI * cutoff_freq / sample_rate;

    let win_taps: Vec<f32> = window(ntaps);
    let mut taps: Vec<f32> = Vec::with_capacity(ntaps);

    let ntaps_i: i32 = ntaps as i32;
    let m: i32 = (ntaps_i - 1) / 2;

    for ix in -m..m + 1 {
        let ix_f: f32 = ix as f32;

        // The windowed sinc function is undefined at zero, so set it to
        // something sensible here.
        if ix == 0 {
            let hi = wc / PI * win_taps[(ix + m) as usize];
            taps.push(hi);
        } else {
            let hi = (ix_f * wc).sin() / (ix_f * PI) *
                win_taps[(ix + m) as usize];
            taps.push(hi);
        }
    }
    taps
}
<commit_msg>Added deviation as in input parameter<commit_after>extern crate num;
extern crate portaudio;
extern crate rtlsdr;

use portaudio as pa;
use rtlsdr::{RTLSDRDevice, RTLSDRError};
use num::{Num, Zero};
use num::complex::Complex;
use std::env;
use std::f32::consts::PI;
use std::error::Error;
use pa::error::Error as PAError;
use std::thread;
use std::sync::mpsc::channel;

// Make sure that the buffer size is radix-2, otherwise the read_sync function
// will fail with an error code of -8. I want the buffer sizes between the SDR
// and the audio sizes to match, but since we get two samples per read the
// buffer size needs to be double.
const SDR_BUF_SIZE: usize = 262_144;

// The buffer size for the audio sink.
const AUDIO_BUF_SIZE: usize = 524_288;

// Most other sample rates fail, but this one works for my particular device.
// I will investigate exactly what's happening here and generate a list of
// possible sample rates (if there are others besides this one).
const SDR_SAMPLE_RATE: u32 = 2_400_000;

// Here is the sample rate of the output waveform we'll try to use.
const AUDIO_SAMPLE_RATE: f32 = 48000.0;

// The number of channels being used for audio.
const CHANNELS: i32 = 2;

// We set the gain manually right now since it seems to be a better option
// than using the AGC (the SDR gets blazing hot with the AGC on).
const SDR_GAIN: i32 = 496;

fn main() {
    match run() {
        Ok(()) => (),
        Err(e) => println!("ERROR: {}", e),
    }
}

struct RTLSDR {
    rtlsdr: RTLSDRDevice,
}

unsafe impl Send for RTLSDR {}

fn run() -> Result<(), Box<Error>> {
    let sdr_index: i32 = env::args()
        .nth(1)
        .expect("Please specify the SDR index.")
        .parse::<i32>()?;
    let fm_freq_mhz: f32 = env::args()
        .nth(2)
        .expect("Please specify a center frequency in MHz.")
        .parse::<f32>()?;
    let bandwidth: u32 = env::args()
        .nth(3)
        .expect("Please specify the bandwidth.")
        .parse::<u32>()?;
    let deviation: f32 = env::args()
        .nth(4)
        .expect("Please specify the deviation.")
        .parse::<f32>()?;

    // Set up the threading channels for communication between our two threads.
    let (dongle_tx, dongle_rx) = channel();
    let (demod_tx, demod_rx) = channel();

    // This thread only collects data off of the SDR as quickly as possible and
    // sends it out.
    thread::spawn(move || {
        // Initialize the SDR based on the user's input parameters.
        let fm_freq: u32 = (fm_freq_mhz * 1e6) as u32;
        let mut sdr =
            RTLSDR { rtlsdr: init_sdr(sdr_index, fm_freq, bandwidth).unwrap() };

        loop {
            let bytes = sdr.rtlsdr.read_sync(SDR_BUF_SIZE).unwrap();
            let iq_vec = samps_to_cmplx(bytes.as_slice()).unwrap();
            dongle_tx.send(iq_vec).unwrap();
        }
    });

    // This thread does all of the signal processing required to demodulate
    // the signal and get it to a nice audio stream.
    thread::spawn(move || {
        // Decimating down to 100k samples seems to work the best. I don't
        // really quite get why that is, so I'll need to do more research.
        let dec_rate_filt = 6;
        let dec_rate_audio = 4;

        let taps_filt = windowed_sinc(
            bandwidth as f32,
            SDR_SAMPLE_RATE as f32,
            16,
            &hamming,
        );
        let taps_audio = windowed_sinc(
            bandwidth as f32 / dec_rate_filt as f32,
            SDR_SAMPLE_RATE as f32 / dec_rate_filt as f32,
            16,
            &hamming,
        );

        let mut prev = Complex::zero();
        loop {
            // First, low_pass and decimate the raw IQ signal.
            let mut iq_vec = dongle_rx.recv().unwrap();
            iq_vec = filter(&iq_vec, &taps_filt);
            iq_vec = decimate(&iq_vec, dec_rate_filt);

            // Next, demodulate the signal and filter the signal again. The
            // second filter seems to help with the noise after demodulation.
            let res = demod_fm(&iq_vec, prev, deviation);
            let mut demod_iq = filter_real(&res.0, &taps_audio);
            demod_iq = decimate(&demod_iq, dec_rate_audio);
            prev = res.1;
            demod_tx.send(demod_iq).unwrap();
        }
    });

    // Initialize the PortAudio class. We use a blocking stream since we're
    // doing processing on the signal and can't keep up with non-blocking
    // timing requirements.
    let audio = try!(pa::PortAudio::new());
    let settings = try!(audio.default_output_stream_settings(
        CHANNELS,
        AUDIO_SAMPLE_RATE as f64,
        AUDIO_BUF_SIZE as u32,
    ));
    let mut stream = try!(audio.open_blocking_stream(settings));
    try!(stream.start());

    loop {
        let demod_iq = demod_rx.recv().unwrap();
        let buffer_frames = (demod_iq.len() / CHANNELS as usize) as u32;
        let mut out_frames = 0;

        while out_frames < buffer_frames {
            out_frames = match stream.write_available() {
                Ok(available) => {
                    match available {
                        pa::StreamAvailable::Frames(frames) => frames as u32,
                        pa::StreamAvailable::InputOverflowed => {
                            return Err(Box::new(PAError::InputOverflowed))
                        }
                        pa::StreamAvailable::OutputUnderflowed => {
                            return Err(Box::new(PAError::OutputUnderflowed))
                        }
                    }
                }
                Err(e) => return Err(Box::new(e)),
            };
        }

        let n_write_samples = buffer_frames as usize * CHANNELS as usize;

        stream.write(
            buffer_frames,
            |output| for ix in 0..n_write_samples {
                output[ix] = demod_iq[ix];
            },
        )?;
    }
}

fn init_sdr(
    sdr_index: i32,
    fm_freq: u32,
    bandwidth: u32,
) -> Result<RTLSDRDevice, RTLSDRError> {
    let mut sdr = try!(rtlsdr::open(sdr_index));
    try!(sdr.set_center_freq(fm_freq));
    try!(sdr.set_sample_rate(SDR_SAMPLE_RATE));
    try!(sdr.set_tuner_bandwidth(bandwidth));
    try!(sdr.set_tuner_gain(SDR_GAIN));
    try!(sdr.set_agc_mode(false));
    try!(sdr.reset_buffer());
    Ok(sdr)
}

/// Decimates a signal by an integer factor.
///
/// # Arguments
///
/// * `signal` - The signal to decimate
/// * `dec_rate` - The decimation factor
///
fn decimate<T: Copy>(signal: &[T], dec_rate: usize) -> Vec<T> {
    let mut ix = 0;
    let new_size = (signal.len() / dec_rate + 1) as usize;
    let mut signal_dec = Vec::<T>::with_capacity(new_size);
    while ix < signal.len() {
        signal_dec.push(signal[ix]);
        ix += dec_rate;
    }
    signal_dec
}

/// Converts samples from the SDR into complex IQ pairs.
///
/// # Arguments
///
/// * `bytes` - The unsigned bytes from the SDR.
///
fn samps_to_cmplx(bytes: &[u8]) -> Option<Vec<Complex<f32>>> {
    // First, check that we've been given an even number of bytes. Not sure
    // what to do if we don't get I and Q.
    let bytes_len = bytes.len();
    if bytes_len % 2 != 0 {
        println!("Received an odd number of samples.");
        return None;

    }

    // Write the values to the complex value and normalize from [0, 255] to
    // [-1, 1].
    let mut iq_vec: Vec<Complex<f32>> = Vec::with_capacity(bytes_len / 2);
    for iq in bytes.chunks(2) {
        let iq_cmplx = Complex::new(
            (iq[0] as f32 - 127.0) / 127.0,
            (iq[1] as f32 - 127.0) / 127.0,
        );
        iq_vec.push(iq_cmplx);
    }
    Some(iq_vec)
}

/// Performs FM demodulation on a complex signal.
///
/// # Arguments
///
/// * `iq` - The IQ samples to demodulate.
/// * `prev` - The previous sample from the previous demodulation.
/// * `deviation` - The maximum frequency deviation in Hz.
///
fn demod_fm(
    iq: &[Complex<f32>],
    prev: Complex<f32>,
    deviation: f32,
) -> (Vec<f32>, Complex<f32>) {

    let mut p = prev;
    let mut demod_queue: Vec<f32> = Vec::with_capacity(iq.len());

    let gain = SDR_SAMPLE_RATE as f32 / (2.0 * PI * deviation);

    for samp in iq {
        let conj = p.conj() * samp;
        demod_queue.push(conj.arg() * gain);
        p = *samp;
    }
    (demod_queue, p)
}

/// Filters a Complex vector and returns a new vector.
///
/// # Arguments
///
/// * `samples` - The complex samples to filter
/// * `taps` - The taps of the filter
///
fn filter<T: Copy + Num + Zero>(
    samples: &[Complex<T>],
    taps: &[T],
) -> Vec<Complex<T>> {

    // We'll lose taps.len() - 1 samples from the filtering.
    let mut filt_samps: Vec<Complex<T>> =
        Vec::with_capacity(samples.len() - taps.len() + 1);

    for window in samples.windows(taps.len()) {
        let iter = window.iter().zip(taps.iter());
        let filt_samp = iter.fold(Complex::zero(), |acc, (x, y)| acc + *x * *y);
        filt_samps.push(filt_samp);
    }
    filt_samps
}

/// Filters a real vector and returns a new vector.
///
/// # Arguments
///
/// * `samples` - The real samples to filter
/// * `taps` - The taps of the filter
///
fn filter_real<T: Copy + Num + Zero>(
    samples: &[T],
    taps: &[T],
) -> Vec<T> {
    // We'll lose taps.len() - 1 samples from the filtering.
    let mut filt_samps: Vec<T> =
        Vec::with_capacity(samples.len() - taps.len() + 1);
    for window in samples.windows(taps.len()) {
        let iter = window.iter().zip(taps.iter());
        let filt_samp = iter.fold(T::zero(), |acc, (x, y)| acc + *x * *y);
        filt_samps.push(filt_samp);
    }
    filt_samps
}

/// Generates a vector containing the taps for a Hamming window.
///
/// # Arguments
///
/// * `ntaps` - The number of taps to generate for the filter
///
fn hamming(ntaps: usize) -> Vec<f32> {
    let m: f32 = ntaps as f32 - 1.0;
    let mut taps: Vec<f32> = Vec::with_capacity(ntaps);
    for ix in 0..ntaps {
        taps.push(0.54 - 0.46 * (2.0 * PI * ix as f32 / m).cos());
    }
    taps
}

/// Generates filter taps for a sinc filter with a windowing function.
///
/// # Arguments
///
/// * `cutoff_freq` - The cutoff frequency in Hertz
/// * `sample_rate` - The sample frequency in Hertz
/// * `ntaps` - The number of taps to generate for the window
/// * `window` - The windowing function used to generate the taps for the window
///
fn windowed_sinc(
    cutoff_freq: f32,
    sample_rate: f32,
    ntaps: usize,
    window: &Fn(usize) -> Vec<f32>,
) -> Vec<f32> {
    // Convert the frequency to radians and make it a ratio of the sample rate.
    let wc: f32 = 2.0 * PI * cutoff_freq / sample_rate;

    let win_taps: Vec<f32> = window(ntaps);
    let mut taps: Vec<f32> = Vec::with_capacity(ntaps);

    let ntaps_i: i32 = ntaps as i32;
    let m: i32 = (ntaps_i - 1) / 2;

    for ix in -m..m + 1 {
        let ix_f: f32 = ix as f32;

        // The windowed sinc function is undefined at zero, so set it to
        // something sensible here.
        if ix == 0 {
            let hi = wc / PI * win_taps[(ix + m) as usize];
            taps.push(hi);
        } else {
            let hi = (ix_f * wc).sin() / (ix_f * PI) *
                win_taps[(ix + m) as usize];
            taps.push(hi);
        }
    }
    taps
}
<|endoftext|>


<commit_before>#![deny(warnings)]

extern crate chan;
extern crate crypto;
extern crate walkdir;

use self::crypto::digest::Digest;
use self::crypto::sha1::Sha1;
use std::env;
use std::fs::File;
use std::io::Read;
use std::path::Path;
use std::io;
use walkdir::WalkDir;
use std::thread;
use std::sync::Arc;

fn calc_hash(p: &Path, hasher: &mut Sha1, buf: &mut [u8]) -> io::Result<String> {
    hasher.reset();
    let mut f = File::open(p)?;
    loop {
        let num_read = f.read(buf)?;
        if num_read == 0 {
            break;
        }
        hasher.input(&buf[..num_read]);
    }
    Ok(hasher.result_str())
}

fn do_it(root: &Path) -> io::Result<()> {
    let jobs = {
        let (s, r) = chan::sync(0);
        let pb = root.to_path_buf();
        thread::spawn(move || {
            for e in WalkDir::new(pb) {
                let e = match e {
                    Ok(e) => e,
                    Err(_) => continue, //? TODO: send Result with error
                };
                if !e.file_type().is_file() {
                    continue;
                }
                s.send(e);
            }
        });
        // This extra lexical scope will drop the initial
        // sender we created. Thus, the channel will be
        // closed when all threads spawned above has completed.
        r
    };
    let wg = chan::WaitGroup::new();
    for _ in 0..8 {
        wg.add(1);
        let wg = wg.clone();
        let jobs = jobs.clone();
        thread::spawn(move || {
            let mut hasher = Sha1::new();
            let mut buf = [0; 1024 * 8];
            for e in jobs {
                let hex = match calc_hash(e.path(), &mut hasher, &mut buf) {
                    Ok(h) => h,
                    Err(_) => continue, //? TODO: Handle error
                };
                println!("{}\t{}", hex, e.path().display());
            }
            wg.done();
        });
    }
    wg.wait();
    Ok(())
}

fn process_dir_entry(e: walkdir::Result<walkdir::DirEntry>) -> i32 {
    let e = match e {
        Ok(e) => e,
        Err(_) => return 0, //? TODO: send Result with error
    };
    if !e.file_type().is_file() {
        return 0;
    }
    1
}

/// factory is needed to allow iterators without std::marker::Send
/// TODO: spawn worker threads
/// TODO: use channels
/// TODO: return channel Receiver as Iterator
fn fan_out_in<F, X, I, T>(factory: F, xform: X) -> i32
    where F: 'static + std::marker::Send + FnOnce() -> T,
          X: 'static + std::marker::Send + std::marker::Sync + Fn(I) -> i32 + 'static,
          I: 'static + std::marker::Send,
          T: IntoIterator<Item = I>
{
    let jobs_rx = {
        let (tx, rx) = chan::sync(0);
        thread::spawn(move || {
            for e in factory().into_iter() {
                tx.send(e);
            }
        });
        rx
    };
    let results_rx = {
        let (tx, rx) = chan::sync(0);
        let xform = Arc::new(xform);
        for _ in 0..8 {
            let tx = tx.clone();
            let jobs_rx = jobs_rx.clone();
            let xform = xform.clone();
            thread::spawn(move || {
                for e in jobs_rx {
                    tx.send(xform(e));
                }
            });
        }
        rx
    };
    let mut i = 0i32;
    for r in results_rx {
        i += r;
    }
    i
}

fn do_it_2(root: &Path) -> io::Result<()> {
    let pb = root.to_path_buf();
    let iter_factory = || WalkDir::new(pb);
    let i = fan_out_in(iter_factory, process_dir_entry);
    println!("{}", i);
    Ok(())
}

fn main() {
    let root = env::args().nth(1).unwrap_or(".".to_string());
    let root = Path::new(root.as_str());
    do_it(root).expect("Hello error 1!");
    do_it_2(root).expect("Hello error 2!");
}
<commit_msg>Rerwrite using generics for scatter-gather.<commit_after>#![deny(warnings)]

extern crate chan;
extern crate crypto;
extern crate walkdir;

use self::crypto::digest::Digest;
use self::crypto::sha1::Sha1;
use std::env;
use std::fs::File;
use std::io::Read;
use std::path::Path;
use std::io;
use walkdir::WalkDir;
use std::thread;
use std::sync::Arc;
use walkdir::DirEntry;

fn calc_hash(p: &Path, hasher: &mut Sha1, buf: &mut [u8]) -> io::Result<String> {
    hasher.reset();
    let mut f = File::open(p)?;
    loop {
        let num_read = f.read(buf)?;
        if num_read == 0 {
            break;
        }
        hasher.input(&buf[..num_read]);
    }
    Ok(hasher.result_str())
}

type Result<T> = std::result::Result<T, TreesumError>;

#[derive(Debug)]
enum TreesumError {
    Io(io::Error),
    Ignored(DirEntry),
    WalkDir(walkdir::Error),
}

fn process_dir_entry(e: walkdir::Result<DirEntry>) -> Result<String> {
    let e = e.map_err(TreesumError::WalkDir)?;
    if !e.file_type().is_file() {
        return Err(TreesumError::Ignored(e));
    }
    let mut hasher = Sha1::new();
    let mut buf = [0; 1024 * 8];
    calc_hash(e.path(), &mut hasher, &mut buf).map_err(TreesumError::Io)
}

fn do_it_2(root: &Path) -> io::Result<()> {
    let pb = root.to_path_buf();
    let iter_factory = || WalkDir::new(pb);
    let results = scatter_gather(iter_factory, process_dir_entry);
    for r in results {
        println!("{:?}", r);
    }
    Ok(())
}

fn main() {
    let root = env::args().nth(1).unwrap_or(".".to_string());
    let root = Path::new(root.as_str());
    do_it_2(root).expect("Hello error 2!");
}

/// The purpose of this newtype is to hide the channel implementation.
struct GatherIter<T>(chan::Iter<T>);
impl<T> Iterator for GatherIter<T> {
    type Item = T;
    fn next(&mut self) -> Option<T> {
        self.0.next()
    }
}

/// factory is needed to allow iterators without std::marker::Send
/// TODO: spawn worker threads
/// TODO: use channels
/// TODO: return channel Receiver as Iterator
fn scatter_gather<F, X, J, R, I>(factory: F, xform: X) -> GatherIter<R>
    where F: 'static + std::marker::Send + FnOnce() -> I,
          X: 'static + std::marker::Send + std::marker::Sync + Fn(J) -> R,
          J: 'static + std::marker::Send,
          R: 'static + std::marker::Send,
          I: IntoIterator<Item = J>
{
    let jobs_rx = {
        let (tx, rx) = chan::sync(0);
        thread::spawn(move || {
            for e in factory().into_iter() {
                tx.send(e);
            }
        });
        rx
    };
    let results_rx = {
        let (tx, rx) = chan::sync(0);
        let xform = Arc::new(xform); // TODO: Investigate why this is needed.
        for _ in 0..8 {
            let tx = tx.clone();
            let jobs_rx = jobs_rx.clone();
            let xform = xform.clone();
            thread::spawn(move || {
                for e in jobs_rx {
                    tx.send(xform(e));
                }
            });
        }
        rx
    };
    GatherIter(results_rx.iter())
}<|endoftext|>


<commit_before>extern crate glowygraph as gg;
extern crate petgraph;
extern crate glium;
extern crate num;
extern crate nalgebra as na;
extern crate zoom;
extern crate rand;
extern crate itertools;
extern crate mli;
use itertools::*;

const SEPARATION_MAGNITUDE: f64 = 0.1;

use na::{ToHomogeneous, Translation, Rotation};
use num::traits::One;

mod bot;
use bot::*;
mod node;
use node::*;
mod rank;
use rank::*;

pub type Vec3 = na::Vec3<f64>;

fn vec_to_spos(v: Vec3) -> [f32; 3] {
    match v {
        Vec3{x, y, z} => [x as f32, y as f32, z as f32]
    }
}

fn main() {
    use glium::DisplayBuild;
    use rand::{SeedableRng, Rng};
    let mut rng = rand::Isaac64Rng::from_seed(&[1, 2, 3, 4]);

    let display = glium::glutin::WindowBuilder::new().with_vsync().build_glium().unwrap();
    let window = display.get_window().unwrap();
    //window.set_cursor_state(glium::glutin::CursorState::Hide).ok().unwrap();
    let glowy = gg::Renderer::new(&display);

    let mut deps = petgraph::Graph::<Node, bool, petgraph::Undirected>::new_undirected();
    deps.add_node(Node::new(5000, zoom::BasicParticle::default()));

    //Set mouse cursor to middle
    {
        let (dimx, dimy) = display.get_framebuffer_dimensions();
        let (hdimx, hdimy) = (dimx/2, dimy/2);
        window.set_cursor_position(hdimx as i32, hdimy as i32).ok().unwrap();
    }

    let perspective = *na::Persp3::new(1.5, 1.0, 0.0, 500.0).to_mat().as_ref();
    let mut movement = na::Iso3::<f32>::one();

    let mut upstate = glium::glutin::ElementState::Released;
    let mut dnstate = glium::glutin::ElementState::Released;
    let mut ltstate = glium::glutin::ElementState::Released;
    let mut rtstate = glium::glutin::ElementState::Released;
    let mut fdstate = glium::glutin::ElementState::Released;
    let mut bkstate = glium::glutin::ElementState::Released;

    loop {
        use glium::Surface;

        let mut target = display.draw();
        target.clear_color(0.0, 0.0, 0.0, 1.0);

        let matr = movement.to_homogeneous() * 3.0;

        //Update forces between nodes
        for i in deps.edge_indices() {
            let node_indices = deps.edge_endpoints(i).unwrap();
            let nodes = deps.index_twice_mut(node_indices.0, node_indices.1);

            //Apply spring forces to keep them together
            zoom::hooke(&nodes.0.particle, &nodes.1.particle, 0.001);
        }

        {
            let nodes = deps.raw_nodes();
            for i in 0..nodes.len() {
                for j in (i+1)..nodes.len() {
                    //Apply repulsion forces to keep them from being too close
                    zoom::gravitate_radius(&nodes[i].weight.particle, &nodes[j].weight.particle, -0.001);
                }
            }
        }

        //Update nodes
        for i in deps.node_indices() {
            if deps.node_weight(i).unwrap().should_split() {
                use std::f64::consts::PI;
                use num::traits::Float;
                let theta = rng.gen_range(0.0, 2.0 * PI);
                let phi = rng.gen_range(-1.0, 1.0).acos();
                let rand_unit_dir = Vec3::new(
                    theta.cos() * phi.sin(),
                    theta.sin() * phi.sin(),
                    phi.cos(),
                );

                //Divide energy in half before splitting
                deps.node_weight_mut(i).unwrap().energy /= 2;

                //Destroy all bots in node
                deps.node_weight_mut(i).unwrap().bots.clear();

                let nnode = {
                    let nref = deps.node_weight(i).unwrap();
                    Node::new(
                        nref.energy,
                        nref.particle.p.clone(),
                    )
                };

                let newindex = deps.add_node(nnode);

                deps.add_edge(i, newindex, false);

                //Add a positive impulse to this particle
                deps.node_weight_mut(i).unwrap().particle.p.velocity =
                    deps.node_weight_mut(i).unwrap().particle.p.velocity +
                    rand_unit_dir * SEPARATION_MAGNITUDE;

                //Add a negative impulse to the other particle
                deps.node_weight_mut(newindex).unwrap().particle.p.velocity =
                    deps.node_weight_mut(newindex).unwrap().particle.p.velocity -
                    rand_unit_dir * SEPARATION_MAGNITUDE;
            }

            deps.node_weight_mut(i).unwrap().advance();
        }

        //Make arrays for bot brain inputs
        let mut node_inputs = [0i64; nodebrain::TOTAL_INPUTS];
        let mut bot_inputs = [0i64; botbrain::TOTAL_INPUTS];
        let mut final_inputs = [0i64; finalbrain::TOTAL_INPUTS];

        //Make the static values
        let statics = [0, 1, 2, -1];
        //Assign static values to each of the input arrays
        node_inputs.iter_mut().set_from(statics.iter().cloned());
        bot_inputs.iter_mut().set_from(statics.iter().cloned());
        final_inputs.iter_mut().set_from(statics.iter().cloned());

        let all_nodes = deps.node_indices().collect_vec();

        //Update bots in nodes
        for i in all_nodes {
            //The current node is always 0; everything else comes after
            let neighbors = std::iter::once(i).chain(deps.neighbors(i)).collect_vec();
            let pnode = deps.node_weight(i).unwrap();

            let mut movers = Vec::<usize>::new();

            //Iterate through all bots (b) in the node being processed
            for (ib, b) in pnode.bots.iter().enumerate() {
                use std::collections::BinaryHeap;
                use mli::SISO;
                //Create a BTree to rank the nodes and fill it with default nodes
                let mut node_heap = BinaryHeap::from(
                    vec![Rank{rank: 0, data: [-1; nodebrain::TOTAL_OUTPUTS]}; finalbrain::TOTAL_NODE_INPUTS]
                );

                //Create a BTree to rank the nodes and fill it with default bots
                let mut bot_heap = BinaryHeap::from(
                    vec![Rank{rank: 0, data: [-1; botbrain::TOTAL_OUTPUTS]}; finalbrain::TOTAL_BOT_INPUTS]
                );

                //Iterate through each node and produce the outputs
                for (i, &n) in neighbors.iter().enumerate() {
                    //Get the node reference
                    let n = deps.node_weight(n).unwrap();
                    //Set the inputs for the node brain
                    node_inputs[4] = n.energy;
                    node_inputs[5] = n.bots.len() as i64;
                    node_inputs[6] = pnode.bots.len() as i64;
                    node_inputs[7] = b.energy;
                    node_inputs[nodebrain::STATIC_INPUTS..].iter_mut().set_from(b.memory.iter().cloned());

                    let mut compute = b.node_brain.compute(&node_inputs[..]);

                    let rank = Rank{
                        rank: compute.next().unwrap(),
                        data: {
                            let mut l = [-1; nodebrain::TOTAL_OUTPUTS];
                            l[0] = i as i64;
                            l[1..].iter_mut().set_from(compute);
                            l
                        },
                    };

                    //Add this rank to the heap
                    node_heap.push(rank);
                    //Remove the lowest rank from the heap to stay at the same amount
                    node_heap.pop();
                }

                //Iterate through each bot and produce the outputs
                for (ib, ob) in pnode.bots.iter().enumerate() {
                    //Set the inputs for the bot brain
                    bot_inputs[4] = pnode.energy;
                    bot_inputs[5] = pnode.bots.len() as i64;
                    bot_inputs[6] = b.energy;
                    bot_inputs[7] = ob.energy;
                    bot_inputs[8] = ob.signal;
                    bot_inputs[botbrain::STATIC_INPUTS..].iter_mut().set_from(b.memory.iter().cloned());

                    let mut compute = b.bot_brain.compute(&bot_inputs[..]);

                    let rank = Rank{
                        rank: compute.next().unwrap(),
                        data: {
                            let mut l = [-1; botbrain::TOTAL_OUTPUTS];
                            l[0] = ib as i64;
                            l[1..].iter_mut().set_from(compute);
                            l
                        },
                    };

                    //Add this rank to the heap
                    bot_heap.push(rank);
                    //Remove the lowest rank from the heap to stay at the same amount
                    bot_heap.pop();
                }

                //Make the bot's final decision

                //Provide static inputs
                final_inputs[4] = pnode.energy;
                final_inputs[5] = pnode.bots.len() as i64;
                final_inputs[6] = b.energy;
                final_inputs[finalbrain::STATIC_INPUTS..].iter_mut().set_from(
                    b.memory.iter().cloned().chain(
                        //Provide the highest ranking node inputs
                        node_heap.iter().flat_map(|r| r.data.iter().cloned())
                    ).chain(
                        //Provide the highest ranking bot inputs
                        bot_heap.iter().flat_map(|r| r.data.iter().cloned())
                    )
                );

                let mut compute = b.final_brain.compute(&final_inputs[..]);
                let mb = &mut deps.node_weight_mut(i).unwrap().bots[ib];
            }
        }

        //Render nodes
        glowy.render_nodes(&mut target, matr.as_ref(), &perspective,
            &deps.node_weights_mut().map(|n|
                gg::Node{
                    position: vec_to_spos(n.particle.p.position),
                    color: n.color(),
                    falloff: 0.25,
                }
            ).collect_vec()[..]);

        //Render edges
        glowy.render_edges(
            &mut target,
            matr.as_ref(),
            &perspective,
            &deps.edge_indices().map(|e| deps.edge_endpoints(e)).flat_map(|n| {
                    let indices = n.unwrap().clone();
                    let nodes = (deps.node_weight(indices.0).unwrap(), deps.node_weight(indices.1).unwrap());
                    std::iter::once(gg::Node{
                        position: vec_to_spos(nodes.0.particle.p.position),
                        color: nodes.0.color(),
                        falloff: 0.25
                    }).chain(
                    std::iter::once(gg::Node{
                        position: vec_to_spos(nodes.1.particle.p.position),
                        color: nodes.1.color(),
                        falloff: 0.25
                    }))
                }
            ).collect_vec()[..]
        );

        target.finish().unwrap();

        for ev in display.poll_events() {
            match ev {
                glium::glutin::Event::Closed => return,
                glium::glutin::Event::KeyboardInput(state, _, Some(glium::glutin::VirtualKeyCode::W)) => {
                    fdstate = state;
                },
                glium::glutin::Event::KeyboardInput(state, _, Some(glium::glutin::VirtualKeyCode::S)) => {
                    bkstate = state;
                },
                glium::glutin::Event::KeyboardInput(state, _, Some(glium::glutin::VirtualKeyCode::A)) => {
                    ltstate = state;
                },
                glium::glutin::Event::KeyboardInput(state, _, Some(glium::glutin::VirtualKeyCode::D)) => {
                    rtstate = state;
                },
                glium::glutin::Event::KeyboardInput(state, _, Some(glium::glutin::VirtualKeyCode::Q)) => {
                    dnstate = state;
                },
                glium::glutin::Event::KeyboardInput(state, _, Some(glium::glutin::VirtualKeyCode::E)) => {
                    upstate = state;
                },
                glium::glutin::Event::MouseMoved((x, y)) => {
                    let (dimx, dimy) = display.get_framebuffer_dimensions();
                    let (hdimx, hdimy) = (dimx/2, dimy/2);
                    movement.append_rotation_mut(&na::Vec3::new(-(y - hdimy as i32) as f32 / 128.0,
                        (x - hdimx as i32) as f32 / 128.0, 0.0));
                    window.set_cursor_position(hdimx as i32, hdimy as i32).ok().unwrap();
                },
                _ => ()
            }
        }

        if upstate == glium::glutin::ElementState::Pressed {
            movement.append_translation_mut(&na::Vec3::new(0.0, -0.1, 0.0));
        }
        if dnstate == glium::glutin::ElementState::Pressed {
            movement.append_translation_mut(&na::Vec3::new(0.0, 0.1, 0.0));
        }
        if ltstate == glium::glutin::ElementState::Pressed {
            movement.append_translation_mut(&na::Vec3::new(-0.1, 0.0, 0.0));
        }
        if rtstate == glium::glutin::ElementState::Pressed {
            movement.append_translation_mut(&na::Vec3::new(0.1, 0.0, 0.0));
        }
        if fdstate == glium::glutin::ElementState::Pressed {
            movement.append_translation_mut(&na::Vec3::new(0.0, 0.0, -0.1));
        }
        if bkstate == glium::glutin::ElementState::Pressed {
            movement.append_translation_mut(&na::Vec3::new(0.0, 0.0, 0.1));
        }
    }
}
<commit_msg>well, it builds now<commit_after>extern crate glowygraph as gg;
extern crate petgraph;
extern crate glium;
extern crate num;
extern crate nalgebra as na;
extern crate zoom;
extern crate rand;
extern crate itertools;
extern crate mli;
use itertools::*;

const SEPARATION_MAGNITUDE: f64 = 0.1;

use na::{ToHomogeneous, Translation, Rotation};
use num::traits::One;

mod bot;
use bot::*;
mod node;
use node::*;
mod rank;
use rank::*;

pub type Vec3 = na::Vec3<f64>;

fn vec_to_spos(v: Vec3) -> [f32; 3] {
    match v {
        Vec3{x, y, z} => [x as f32, y as f32, z as f32]
    }
}

fn main() {
    use glium::DisplayBuild;
    use rand::{SeedableRng, Rng};
    let mut rng = rand::Isaac64Rng::from_seed(&[1, 2, 3, 4]);

    let display = glium::glutin::WindowBuilder::new().with_vsync().build_glium().unwrap();
    let window = display.get_window().unwrap();
    //window.set_cursor_state(glium::glutin::CursorState::Hide).ok().unwrap();
    let glowy = gg::Renderer::new(&display);

    let mut deps = petgraph::Graph::<Node, bool, petgraph::Undirected>::new_undirected();
    deps.add_node(Node::new(5000, zoom::BasicParticle::default()));

    //Set mouse cursor to middle
    {
        let (dimx, dimy) = display.get_framebuffer_dimensions();
        let (hdimx, hdimy) = (dimx/2, dimy/2);
        window.set_cursor_position(hdimx as i32, hdimy as i32).ok().unwrap();
    }

    let perspective = *na::Persp3::new(1.5, 1.0, 0.0, 500.0).to_mat().as_ref();
    let mut movement = na::Iso3::<f32>::one();

    let mut upstate = glium::glutin::ElementState::Released;
    let mut dnstate = glium::glutin::ElementState::Released;
    let mut ltstate = glium::glutin::ElementState::Released;
    let mut rtstate = glium::glutin::ElementState::Released;
    let mut fdstate = glium::glutin::ElementState::Released;
    let mut bkstate = glium::glutin::ElementState::Released;

    loop {
        use glium::Surface;

        let mut target = display.draw();
        target.clear_color(0.0, 0.0, 0.0, 1.0);

        let matr = movement.to_homogeneous() * 3.0;

        //Update forces between nodes
        for i in deps.edge_indices() {
            let node_indices = deps.edge_endpoints(i).unwrap();
            let nodes = deps.index_twice_mut(node_indices.0, node_indices.1);

            //Apply spring forces to keep them together
            zoom::hooke(&nodes.0.particle, &nodes.1.particle, 0.001);
        }

        {
            let nodes = deps.raw_nodes();
            for i in 0..nodes.len() {
                for j in (i+1)..nodes.len() {
                    //Apply repulsion forces to keep them from being too close
                    zoom::gravitate_radius(&nodes[i].weight.particle, &nodes[j].weight.particle, -0.001);
                }
            }
        }

        //Update nodes
        for i in deps.node_indices() {
            if deps.node_weight(i).unwrap().should_split() {
                use std::f64::consts::PI;
                use num::traits::Float;
                let theta = rng.gen_range(0.0, 2.0 * PI);
                let phi = rng.gen_range(-1.0, 1.0).acos();
                let rand_unit_dir = Vec3::new(
                    theta.cos() * phi.sin(),
                    theta.sin() * phi.sin(),
                    phi.cos(),
                );

                //Divide energy in half before splitting
                deps.node_weight_mut(i).unwrap().energy /= 2;

                //Destroy all bots in node
                deps.node_weight_mut(i).unwrap().bots.clear();

                let nnode = {
                    let nref = deps.node_weight(i).unwrap();
                    Node::new(
                        nref.energy,
                        nref.particle.p.clone(),
                    )
                };

                let newindex = deps.add_node(nnode);

                deps.add_edge(i, newindex, false);

                //Add a positive impulse to this particle
                deps.node_weight_mut(i).unwrap().particle.p.velocity =
                    deps.node_weight_mut(i).unwrap().particle.p.velocity +
                    rand_unit_dir * SEPARATION_MAGNITUDE;

                //Add a negative impulse to the other particle
                deps.node_weight_mut(newindex).unwrap().particle.p.velocity =
                    deps.node_weight_mut(newindex).unwrap().particle.p.velocity -
                    rand_unit_dir * SEPARATION_MAGNITUDE;
            }

            deps.node_weight_mut(i).unwrap().advance();
        }

        //Make arrays for bot brain inputs
        let mut node_inputs = [0i64; nodebrain::TOTAL_INPUTS];
        let mut bot_inputs = [0i64; botbrain::TOTAL_INPUTS];
        let mut final_inputs = [0i64; finalbrain::TOTAL_INPUTS];

        //Make the static values
        let statics = [0, 1, 2, -1];
        //Assign static values to each of the input arrays
        node_inputs.iter_mut().set_from(statics.iter().cloned());
        bot_inputs.iter_mut().set_from(statics.iter().cloned());
        final_inputs.iter_mut().set_from(statics.iter().cloned());

        let all_nodes = deps.node_indices().collect_vec();

        //Update bots in nodes
        for i in all_nodes {
            //The current node is always 0; everything else comes after
            let neighbors = std::iter::once(i).chain(deps.neighbors(i)).collect_vec();

            let mut movers = Vec::<usize>::new();

            //Iterate through all bots (b) in the node being processed
            for ib in 0..deps.node_weight(i).unwrap().bots.len() {
                use std::collections::BinaryHeap;
                use mli::SISO;
                {
                    let pnode = deps.node_weight(i).unwrap();
                    //Create a BTree to rank the nodes and fill it with default nodes
                    let mut node_heap = BinaryHeap::from(
                        vec![Rank{rank: 0, data: [-1; nodebrain::TOTAL_OUTPUTS]}; finalbrain::TOTAL_NODE_INPUTS]
                    );

                    //Create a BTree to rank the nodes and fill it with default bots
                    let mut bot_heap = BinaryHeap::from(
                        vec![Rank{rank: 0, data: [-1; botbrain::TOTAL_OUTPUTS]}; finalbrain::TOTAL_BOT_INPUTS]
                    );

                    //Iterate through each node and produce the outputs
                    for (i, &n) in neighbors.iter().enumerate() {
                        //Get the node reference
                        let n = deps.node_weight(n).unwrap();
                        //Set the inputs for the node brain
                        node_inputs[4] = n.energy;
                        node_inputs[5] = n.bots.len() as i64;
                        node_inputs[6] = pnode.bots.len() as i64;
                        node_inputs[7] = pnode.bots[ib].energy;
                        node_inputs[nodebrain::STATIC_INPUTS..].iter_mut().set_from(pnode.bots[ib].memory.iter().cloned());

                        let mut compute = pnode.bots[ib].node_brain.compute(&node_inputs[..]);

                        let rank = Rank{
                            rank: compute.next().unwrap(),
                            data: {
                                let mut l = [-1; nodebrain::TOTAL_OUTPUTS];
                                l[0] = i as i64;
                                l[1..].iter_mut().set_from(compute);
                                l
                            },
                        };

                        //Add this rank to the heap
                        node_heap.push(rank);
                        //Remove the lowest rank from the heap to stay at the same amount
                        node_heap.pop();
                    }

                    //Iterate through each bot and produce the outputs
                    for (iob, ob) in pnode.bots.iter().enumerate() {
                        //Set the inputs for the bot brain
                        bot_inputs[4] = pnode.energy;
                        bot_inputs[5] = pnode.bots.len() as i64;
                        bot_inputs[6] = pnode.bots[ib].energy;
                        bot_inputs[7] = ob.energy;
                        bot_inputs[8] = ob.signal;
                        bot_inputs[botbrain::STATIC_INPUTS..].iter_mut().set_from(pnode.bots[ib].memory.iter().cloned());

                        let mut compute = pnode.bots[ib].bot_brain.compute(&bot_inputs[..]);

                        let rank = Rank{
                            rank: compute.next().unwrap(),
                            data: {
                                let mut l = [-1; botbrain::TOTAL_OUTPUTS];
                                l[0] = iob as i64;
                                l[1..].iter_mut().set_from(compute);
                                l
                            },
                        };

                        //Add this rank to the heap
                        bot_heap.push(rank);
                        //Remove the lowest rank from the heap to stay at the same amount
                        bot_heap.pop();
                    }

                    //Make the bot's final decision

                    //Provide static inputs
                    final_inputs[4] = pnode.energy;
                    final_inputs[5] = pnode.bots.len() as i64;
                    final_inputs[6] = pnode.bots[ib].energy;
                    final_inputs[finalbrain::STATIC_INPUTS..].iter_mut().set_from(
                        pnode.bots[ib].memory.iter().cloned().chain(
                            //Provide the highest ranking node inputs
                            node_heap.iter().flat_map(|r| r.data.iter().cloned())
                        ).chain(
                            //Provide the highest ranking bot inputs
                            bot_heap.iter().flat_map(|r| r.data.iter().cloned())
                        )
                    );
                }

                let mb = &mut deps.node_weight_mut(i).unwrap().bots[ib];
                let mut compute = mb.final_brain.compute(&final_inputs[..]);
            }
        }

        //Render nodes
        glowy.render_nodes(&mut target, matr.as_ref(), &perspective,
            &deps.node_weights_mut().map(|n|
                gg::Node{
                    position: vec_to_spos(n.particle.p.position),
                    color: n.color(),
                    falloff: 0.25,
                }
            ).collect_vec()[..]);

        //Render edges
        glowy.render_edges(
            &mut target,
            matr.as_ref(),
            &perspective,
            &deps.edge_indices().map(|e| deps.edge_endpoints(e)).flat_map(|n| {
                    let indices = n.unwrap().clone();
                    let nodes = (deps.node_weight(indices.0).unwrap(), deps.node_weight(indices.1).unwrap());
                    std::iter::once(gg::Node{
                        position: vec_to_spos(nodes.0.particle.p.position),
                        color: nodes.0.color(),
                        falloff: 0.25
                    }).chain(
                    std::iter::once(gg::Node{
                        position: vec_to_spos(nodes.1.particle.p.position),
                        color: nodes.1.color(),
                        falloff: 0.25
                    }))
                }
            ).collect_vec()[..]
        );

        target.finish().unwrap();

        for ev in display.poll_events() {
            match ev {
                glium::glutin::Event::Closed => return,
                glium::glutin::Event::KeyboardInput(state, _, Some(glium::glutin::VirtualKeyCode::W)) => {
                    fdstate = state;
                },
                glium::glutin::Event::KeyboardInput(state, _, Some(glium::glutin::VirtualKeyCode::S)) => {
                    bkstate = state;
                },
                glium::glutin::Event::KeyboardInput(state, _, Some(glium::glutin::VirtualKeyCode::A)) => {
                    ltstate = state;
                },
                glium::glutin::Event::KeyboardInput(state, _, Some(glium::glutin::VirtualKeyCode::D)) => {
                    rtstate = state;
                },
                glium::glutin::Event::KeyboardInput(state, _, Some(glium::glutin::VirtualKeyCode::Q)) => {
                    dnstate = state;
                },
                glium::glutin::Event::KeyboardInput(state, _, Some(glium::glutin::VirtualKeyCode::E)) => {
                    upstate = state;
                },
                glium::glutin::Event::MouseMoved((x, y)) => {
                    let (dimx, dimy) = display.get_framebuffer_dimensions();
                    let (hdimx, hdimy) = (dimx/2, dimy/2);
                    movement.append_rotation_mut(&na::Vec3::new(-(y - hdimy as i32) as f32 / 128.0,
                        (x - hdimx as i32) as f32 / 128.0, 0.0));
                    window.set_cursor_position(hdimx as i32, hdimy as i32).ok().unwrap();
                },
                _ => ()
            }
        }

        if upstate == glium::glutin::ElementState::Pressed {
            movement.append_translation_mut(&na::Vec3::new(0.0, -0.1, 0.0));
        }
        if dnstate == glium::glutin::ElementState::Pressed {
            movement.append_translation_mut(&na::Vec3::new(0.0, 0.1, 0.0));
        }
        if ltstate == glium::glutin::ElementState::Pressed {
            movement.append_translation_mut(&na::Vec3::new(-0.1, 0.0, 0.0));
        }
        if rtstate == glium::glutin::ElementState::Pressed {
            movement.append_translation_mut(&na::Vec3::new(0.1, 0.0, 0.0));
        }
        if fdstate == glium::glutin::ElementState::Pressed {
            movement.append_translation_mut(&na::Vec3::new(0.0, 0.0, -0.1));
        }
        if bkstate == glium::glutin::ElementState::Pressed {
            movement.append_translation_mut(&na::Vec3::new(0.0, 0.0, 0.1));
        }
    }
}
<|endoftext|>


<commit_before>#![feature(io, path, env, os, core)]

use std::env;
use std::str::StrExt;
use std::old_io::File;
use tokenizer::Tokenizer;
use interpreter::Interpreter;

mod tokenizer;
mod operations;
mod interpreter;

fn main() {
    let mut arguments = env::args();
    arguments.next();
    match arguments.next() {
        Some(filename) => {
            match File::open(&Path::new(filename)).read_to_string() {
                Ok(s) => {
                    let content = s.replace("\n", " ");
                    let words = content.split(' ').collect::<Vec<&str>>();
                    let mut tokenizer = Tokenizer::new(words);
                    let ops = tokenizer.tokenize();
                    let mut interpreter = Interpreter::new(ops);
                    interpreter.run();
                },
                Err(error) => panic!("{}", error),
            }
        },
        None => println!("Usage: groot file.groot"),
    }
}
<commit_msg>remove warnings from io feature<commit_after>#![feature(old_io, path, env, os, core)]

use std::env;
use std::str::StrExt;
use std::old_io::File;
use tokenizer::Tokenizer;
use interpreter::Interpreter;

mod tokenizer;
mod operations;
mod interpreter;

fn main() {
    let mut arguments = env::args();
    arguments.next();
    match arguments.next() {
        Some(filename) => {
            match File::open(&Path::new(filename)).read_to_string() {
                Ok(s) => {
                    let content = s.replace("\n", " ");
                    let words = content.split(' ').collect::<Vec<&str>>();
                    let mut tokenizer = Tokenizer::new(words);
                    let ops = tokenizer.tokenize();
                    let mut interpreter = Interpreter::new(ops);
                    interpreter.run();
                },
                Err(error) => panic!("{}", error),
            }
        },
        None => println!("Usage: groot file.groot"),
    }
}
<|endoftext|>


<commit_before>#![allow(non_upper_case_globals)]

use std::ffi::OsStr;
use std::path::PathBuf;

fn home() -> PathBuf {
    std::env::home_dir().unwrap()
}

fn sbt_launch_dir() -> PathBuf {
    let mut p = PathBuf::from(home());
    p.push(".sbt/launchers");
    p
}

fn build_props() -> PathBuf {
    PathBuf::from("project/build.properties")
}

fn build_props_sbt() -> String {
    if let Ok(f) = std::fs::File::open(build_props()) {
        let f = std::io::BufReader::new(f);

        use std::io::BufRead;
        for line in f.lines() {
            let line = line.unwrap();
            if line.starts_with("sbt.version") {
                let line = line.replace("=", " ");
                let line = line.replace("\r", " ");
                return line.split_whitespace().nth(1).unwrap().to_owned();
            }
        }
    }
    "".to_owned()
}

fn jar_file(sbt_version: &str) -> PathBuf {
    let mut p = PathBuf::from(sbt_launch_dir());
    p.push(sbt_version);
    p.push("sbt-launch.jar");
    p
}

fn exec_runner<S: AsRef<OsStr>>(args: &[S]) {
    use std::os::unix::process::CommandExt;
    let err = std::process::Command::new(&args[0]).args(&args[1..]).exec();
    println!("error: {}", err);
    if let Some(err) = err.raw_os_error() {
        std::process::exit(err);
    }
    std::process::exit(-1)
}

fn main() {
    let sbt_version = build_props_sbt();
    let sbt_jar = jar_file(&sbt_version);

    let java_cmd = "java";
    let extra_jvm_opts = ["-Xms512m", "-Xmx1536m", "-Xss2m"];
    let java_args: [&str; 0] = [];
    let sbt_commands: [&str; 0] = [];
    let residual_args = ["shell"];

    let mut exec_args: Vec<&OsStr> = Vec::new();
    exec_args.push(java_cmd.as_ref());
    exec_args.extend_from_slice(&extra_jvm_opts.iter().map(|x| x.as_ref()).collect::<Vec<_>>());
    exec_args.extend_from_slice(&java_args.iter().map(|x| x.as_ref()).collect::<Vec<_>>());
    exec_args.extend_from_slice(&["-jar".as_ref(), sbt_jar.as_ref()]);
    exec_args.extend_from_slice(&sbt_commands.iter().map(|x| x.as_ref()).collect::<Vec<_>>());
    exec_args.extend_from_slice(&residual_args.iter().map(|x| x.as_ref()).collect::<Vec<_>>());
    let exec_args = exec_args;

    exec_runner(&exec_args)
}
<commit_msg>Nest exec_args<commit_after>#![allow(non_upper_case_globals)]

use std::ffi::OsStr;
use std::path::PathBuf;

fn home() -> PathBuf {
    std::env::home_dir().unwrap()
}

fn sbt_launch_dir() -> PathBuf {
    let mut p = PathBuf::from(home());
    p.push(".sbt/launchers");
    p
}

fn build_props() -> PathBuf {
    PathBuf::from("project/build.properties")
}

fn build_props_sbt() -> String {
    if let Ok(f) = std::fs::File::open(build_props()) {
        let f = std::io::BufReader::new(f);

        use std::io::BufRead;
        for line in f.lines() {
            let line = line.unwrap();
            if line.starts_with("sbt.version") {
                let line = line.replace("=", " ");
                let line = line.replace("\r", " ");
                return line.split_whitespace().nth(1).unwrap().to_owned();
            }
        }
    }
    "".to_owned()
}

fn jar_file(version: &str) -> PathBuf {
    let mut p = PathBuf::from(sbt_launch_dir());
    p.push(version);
    p.push("sbt-launch.jar");
    p
}

fn exec_runner<S: AsRef<OsStr>>(args: &[S]) {
    use std::os::unix::process::CommandExt;
    let err = std::process::Command::new(&args[0]).args(&args[1..]).exec();
    println!("error: {}", err);
    if let Some(err) = err.raw_os_error() {
        std::process::exit(err);
    }
    std::process::exit(-1)
}

fn main() {
    let sbt_version = build_props_sbt();
    let sbt_jar = jar_file(&sbt_version);

    let java_cmd = "java";
    let extra_jvm_opts = ["-Xms512m", "-Xmx1536m", "-Xss2m"];
    let java_args: [&str; 0] = [];
    let sbt_commands: [&str; 0] = [];
    let residual_args = ["shell"];

    let exec_args: Vec<&OsStr> = {
        let mut exec_args = Vec::new();
        exec_args.push(java_cmd.as_ref());
        exec_args.extend_from_slice(&extra_jvm_opts.iter().map(|x| x.as_ref()).collect::<Vec<_>>());
        exec_args.extend_from_slice(&java_args.iter().map(|x| x.as_ref()).collect::<Vec<_>>());
        exec_args.extend_from_slice(&["-jar".as_ref(), sbt_jar.as_ref()]);
        exec_args.extend_from_slice(&sbt_commands.iter().map(|x| x.as_ref()).collect::<Vec<_>>());
        exec_args.extend_from_slice(&residual_args.iter().map(|x| x.as_ref()).collect::<Vec<_>>());
        exec_args
    };

    exec_runner(&exec_args)
}
<|endoftext|>


<commit_before>#![feature(semaphore)]
#![feature(read_exact)]

extern crate irc;
extern crate rustc_serialize;
extern crate toml;
extern crate byteorder;

mod playpen;
mod cfg;
mod util;
mod eval;

use irc::client::prelude::*;
use std::sync::{Arc, Mutex, Semaphore};
use std::thread;
use std::collections::{VecDeque, HashMap};

use cfg::EvalbotCfg;
use eval::Req;

macro_rules! send_msg {
    ($conn:expr, $dest:expr, $line:expr) => {
        match $conn.send_privmsg($dest, $line) {
            Err(x) => println!("failed to send message: {}", x),
            _ => ()
        }
    };
}

#[derive(Clone)]
struct State {
    requests: Arc<Mutex<VecDeque<Req>>>,
    has_work: Arc<Semaphore>,
    cfg: Arc<EvalbotCfg>,
    evaluators: Arc<HashMap<eval::Lang, Box<eval::Evaluator>>>
}

fn evaluate_loop<'a, S, T, U>(conn: Arc<S>, state: State) -> !
    where T: IrcRead, U: IrcWrite, S: ServerExt<'a, T, U> + Sized {
    let has_work = state.has_work;
    let cfg = state.cfg;
    let requests = state.requests;
    loop {
        let work;
        {
            has_work.acquire();
            let mut rvec = requests.lock().unwrap();
            work = rvec.pop_front();
            std::mem::drop(rvec);
        }

        if let Some(work) = work {
            let result = state.evaluators.get(&work.language).unwrap()
                .eval(&work.code, &cfg.sandbox_dir, cfg.playpen_timeout);

            let (result, err) = match result {
                Ok(x) => (x, false),
                Err(x) => (x, true)
            };
            let result = util::wrap_output(&result,
                                     if work.is_channel { cfg.max_channel_line_len }
                                     else { 425 });

            // TODO: gist the overflow
            let max_lines = if work.is_channel { cfg.max_channel_lines } else { cfg.max_priv_lines };
            let result = util::truncate_output(result, max_lines);

            let dest = if work.is_channel { &work.target } else { &work.sender };
            for line in result.1.iter() {
                let line = format!("{}{}", if work.is_channel && !err { &cfg.chan_output_prefix as &str } else { "" }, line);
                send_msg!(conn, dest, &line);
            }
            if result.0 {
                send_msg!(conn, dest, "(output truncated)");
            }
        }
    }
}

fn parse_msg(message: &Message) -> Option<Req> {
    let sender = message.get_source_nickname().unwrap_or("");
    if let Ok(Command::PRIVMSG(target, message)) = message.into() {
        let in_channel = target.starts_with('#');
        if !in_channel && message.contains('\x01') {
            // we don't accept code via CTCP
            return None;
        }

        let tok: Vec<&str> = message.trim().splitn(2, '>').collect();
        match tok.len() {
            2 => (),
            _ => return None
        };
        let lang = match tok[0].parse::<eval::Lang>() {
            Ok(x) => x,
            Err(_) => return None
        };
        Some(Req {
            is_channel: in_channel,
            sender: sender.to_owned(),
            target: target.to_owned(),
            code: tok[1].to_owned(),
            language: lang
        })
    } else {
        None
    }
}

fn main() {
    let config = match cfg::read("evalbot.toml") {
        Ok(x) => x,
        Err(x) => panic!("could not read config; {}", x)
    };
    println!("read config: {:?}", config);

    let conn = Arc::new(IrcServer::from_config(config.irc_config.clone()).unwrap());
    let config = Arc::new(config);
    let evalreqs = Arc::new(Mutex::new(VecDeque::new()));
    let has_work = Arc::new(Semaphore::new(0));

    let mut evaluators = HashMap::new();
    evaluators.insert(eval::Lang::Rust, eval::evaluator(eval::Lang::Rust, &config.sandbox_dir));
    evaluators.insert(eval::Lang::RustRaw, eval::evaluator(eval::Lang::RustRaw, &config.sandbox_dir));
    evaluators.insert(eval::Lang::Python, eval::evaluator(eval::Lang::Python, &config.sandbox_dir));
    evaluators.insert(eval::Lang::CSharp, eval::evaluator(eval::Lang::CSharp, &config.sandbox_dir));
    let evaluators = Arc::new(evaluators);

    let state = State {
        cfg: config,
        requests: evalreqs,
        has_work: has_work,
        evaluators: evaluators
    };
    for _ in 0..state.cfg.eval_threads {
        let conn = conn.clone();
        let state = state.clone();
        thread::spawn(move || { evaluate_loop(conn, state); });
    }
    loop {
        conn.identify().unwrap();
        for maybe_msg in conn.iter() {
            let msg = match maybe_msg {
                Ok(x) => x,
                Err(x) => {
                    println!("{}, reconnecting", x);
                    break
                }
            };

            let req = parse_msg(&msg);
            if let Some(x) = req {
                println!("{} @ {} {:?}: {}", x.sender, x.target, x.language, x.code);
                state.requests.lock().unwrap().push_back(x); // if mutex is poisoned, just bail
                state.has_work.release();
            }
        }
        conn.reconnect().unwrap();
    }
}
<commit_msg>Truncate notice on last line; say something if no output<commit_after>#![feature(semaphore)]
#![feature(read_exact)]

extern crate irc;
extern crate rustc_serialize;
extern crate toml;
extern crate byteorder;

mod playpen;
mod cfg;
mod util;
mod eval;

use irc::client::prelude::*;
use std::sync::{Arc, Mutex, Semaphore};
use std::thread;
use std::collections::{VecDeque, HashMap};

use cfg::EvalbotCfg;
use eval::Req;

macro_rules! send_msg {
    ($conn:expr, $dest:expr, $line:expr) => {
        match $conn.send_privmsg($dest, $line) {
            Err(x) => println!("failed to send message: {}", x),
            _ => ()
        }
    };
}

#[derive(Clone)]
struct State {
    requests: Arc<Mutex<VecDeque<Req>>>,
    has_work: Arc<Semaphore>,
    cfg: Arc<EvalbotCfg>,
    evaluators: Arc<HashMap<eval::Lang, Box<eval::Evaluator>>>
}

fn evaluate_loop<'a, S, T, U>(conn: Arc<S>, state: State) -> !
    where T: IrcRead, U: IrcWrite, S: ServerExt<'a, T, U> + Sized {
    let has_work = state.has_work;
    let cfg = state.cfg;
    let requests = state.requests;
    loop {
        let work;
        {
            has_work.acquire();
            let mut rvec = requests.lock().unwrap();
            work = rvec.pop_front();
            std::mem::drop(rvec);
        }

        if let Some(work) = work {
            let result = state.evaluators.get(&work.language).unwrap()
                .eval(&work.code, &cfg.sandbox_dir, cfg.playpen_timeout);

            let (result, err) = match result {
                Ok(x) => (x, false),
                Err(x) => (x, true)
            };

            let dest = if work.is_channel { &work.target } else { &work.sender };

            if result.len() == 0 {
                send_msg!(conn, dest, "ok (no output)");
                continue;
            }

            let result = util::wrap_output(&result,
                                     if work.is_channel { cfg.max_channel_line_len }
                                     else { 425 });

            let max_lines = if work.is_channel { cfg.max_channel_lines } else { cfg.max_priv_lines };
            let (truncated, result) = util::truncate_output(result, max_lines);
            let iter = result.iter().take(if truncated {
                // truncated: we take up to the 2nd last line, as we add "... (output truncated)"
                // to the last line
                result.len() - 1
            } else {
                result.len()
            });
            let prefix = if work.is_channel && !err { &cfg.chan_output_prefix as &str } else { "" };
            for line in iter {
                let line = format!("{}{}", prefix, line);
                send_msg!(conn, dest, &line);
            }
            if truncated {
                send_msg!(conn, dest, &format!("{}{}... (output truncated)", prefix, result.last().unwrap()));
            }
        }
    }
}

fn parse_msg(message: &Message) -> Option<Req> {
    let sender = message.get_source_nickname().unwrap_or("");
    if let Ok(Command::PRIVMSG(target, message)) = message.into() {
        let in_channel = target.starts_with('#');
        if !in_channel && message.contains('\x01') {
            // we don't accept code via CTCP
            return None;
        }

        let tok: Vec<&str> = message.trim().splitn(2, '>').collect();
        match tok.len() {
            2 => (),
            _ => return None
        };
        let lang = match tok[0].parse::<eval::Lang>() {
            Ok(x) => x,
            Err(_) => return None
        };
        Some(Req {
            is_channel: in_channel,
            sender: sender.to_owned(),
            target: target.to_owned(),
            code: tok[1].to_owned(),
            language: lang
        })
    } else {
        None
    }
}

fn main() {
    let config = match cfg::read("evalbot.toml") {
        Ok(x) => x,
        Err(x) => panic!("could not read config; {}", x)
    };
    println!("read config: {:?}", config);

    let conn = Arc::new(IrcServer::from_config(config.irc_config.clone()).unwrap());
    let config = Arc::new(config);
    let evalreqs = Arc::new(Mutex::new(VecDeque::new()));
    let has_work = Arc::new(Semaphore::new(0));

    let mut evaluators = HashMap::new();
    evaluators.insert(eval::Lang::Rust, eval::evaluator(eval::Lang::Rust, &config.sandbox_dir));
    evaluators.insert(eval::Lang::RustRaw, eval::evaluator(eval::Lang::RustRaw, &config.sandbox_dir));
    evaluators.insert(eval::Lang::Python, eval::evaluator(eval::Lang::Python, &config.sandbox_dir));
    evaluators.insert(eval::Lang::CSharp, eval::evaluator(eval::Lang::CSharp, &config.sandbox_dir));
    let evaluators = Arc::new(evaluators);

    let state = State {
        cfg: config,
        requests: evalreqs,
        has_work: has_work,
        evaluators: evaluators
    };
    for _ in 0..state.cfg.eval_threads {
        let conn = conn.clone();
        let state = state.clone();
        thread::spawn(move || { evaluate_loop(conn, state); });
    }
    loop {
        conn.identify().unwrap();
        for maybe_msg in conn.iter() {
            let msg = match maybe_msg {
                Ok(x) => x,
                Err(x) => {
                    println!("{}, reconnecting", x);
                    break
                }
            };

            let req = parse_msg(&msg);
            if let Some(x) = req {
                println!("{} @ {} {:?}: {}", x.sender, x.target, x.language, x.code);
                state.requests.lock().unwrap().push_back(x); // if mutex is poisoned, just bail
                state.has_work.release();
            }
        }
        conn.reconnect().unwrap();
    }
}
<|endoftext|>