<commit_before>extern crate num;
extern crate portaudio;
extern crate rtlsdr;
extern crate dft;
extern crate gnuplot;

use portaudio as pa;
use rtlsdr::{RTLSDRDevice, RTLSDRError};
use num::{Num, Zero};
use num::complex::Complex;
use std::env;
use std::f32::consts::PI;
use std::error::Error;
use pa::error::Error as PAError;
use std::thread;
use std::sync::mpsc::channel;

// Make sure that the buffer size is radix-2, otherwise the read_sync function
// will fail with an error code of -8. I want the buffer sizes between the SDR
// and the audio sizes to match, but since we get two samples per read the
// buffer size needs to be double.
const SDR_BUF_SIZE: usize = 524288;

// The buffer size for the audio sink.
const AUDIO_BUF_SIZE: usize = 262144;

// Most other sample rates fail, but this one works for my particular device.
// I will investigate exactly what's happening here and generate a list of
// possible sample rates (if there are others besides this one).
const SDR_SAMPLE_RATE: u32 = 2400000;

// Here is the sample rate of the output waveform we'll try to use.
const AUDIO_SAMPLE_RATE: f32 = 48000.0;

// The number of channels being used for audio.
const CHANNELS: i32 = 2;

fn main() {
    match run() {
        Ok(()) => (),
        Err(e) => println!("ERROR: {}", e),
    }
}

struct RTLSDR {
    rtlsdr: RTLSDRDevice,
}

unsafe impl Send for RTLSDR {}

fn run() -> Result<(), Box<Error>> {
    let sdr_index: i32 = env::args()
        .nth(1)
        .expect("Please specify the SDR index.")
        .parse::<i32>()?;
    let fm_freq_mhz: f32 = env::args()
        .nth(2)
        .expect("Please specify a center frequency in MHz.")
        .parse::<f32>()?;
    let bandwidth: u32 = env::args()
        .nth(3)
        .expect("Please specify the bandwidth.")
        .parse::<u32>()?;

    // Set up the threading channels for communication between our two threads.
    let (dongle_tx, dongle_rx) = channel();
    let (demod_tx, demod_rx) = channel();

    // This thread only collects data off of the SDR as quickly as possible and
    // sends it out.
    thread::spawn(move || {
        // Initialize the SDR based on the user's input parameters.
        let fm_freq: u32 = (fm_freq_mhz * 1e6) as u32;
        let mut sdr =
            RTLSDR { rtlsdr: init_sdr(sdr_index, fm_freq, bandwidth).unwrap() };

        loop {
            let bytes = sdr.rtlsdr.read_sync(SDR_BUF_SIZE).unwrap();
            let iq_vec = read_samples(bytes).unwrap();
            dongle_tx.send(iq_vec).unwrap();
        }
    });

    // This thread does all of the signal processing required to demodulate
    // the signal and get it to a nice audio stream.
    thread::spawn(move || {
        // Decimating down to 100k samples seems to work the best. I don't
        // really quite get why that is, so I'll need to do more research.
        let dec_rate_filt = 6;
        let dec_rate_audio = 4;

        let taps_filt =
            windowed_sinc(bandwidth as f32, SDR_SAMPLE_RATE as f32, 128);
        let taps_audio = windowed_sinc(
            bandwidth as f32 / dec_rate_filt as f32,
            SDR_SAMPLE_RATE as f32 / dec_rate_filt as f32,
            128,
        );

        let mut prev = Complex::zero();
        loop {
            // First, low_pass and decimate the raw IQ signal.
            let mut iq_vec = dongle_rx.recv().unwrap();
            iq_vec = filter(&iq_vec, &taps_filt);
            iq_vec = decimate(iq_vec, dec_rate_filt);

            // Next, demodulate the signal and filter the signal again. The
            // second filter seems to help with the noise after demodulation.
            let res = demod_fm(iq_vec, prev);
            let mut demod_iq = filter_real(&res.0, &taps_audio);
            demod_iq = decimate(demod_iq, dec_rate_audio);
            prev = res.1;
            demod_tx.send(demod_iq).unwrap();
        }
    });

    // Initialize the PortAudio class. We use a blocking stream since we're
    // doing processing on the signal and can't keep up with non-blocking
    // timing requirements.
    let audio = try!(pa::PortAudio::new());
    let settings = try!(audio.default_output_stream_settings(
        CHANNELS,
        AUDIO_SAMPLE_RATE as f64,
        AUDIO_BUF_SIZE as u32,
    ));
    let mut stream = try!(audio.open_blocking_stream(settings));
    try!(stream.start());

    loop {
        let demod_iq = demod_rx.recv().unwrap();

        let out_frames = match stream.write_available() {
            Ok(available) => {
                match available {
                    pa::StreamAvailable::Frames(frames) => frames as u32,
                    pa::StreamAvailable::InputOverflowed => {
                        return Err(Box::new(PAError::InputOverflowed))
                    }
                    pa::StreamAvailable::OutputUnderflowed => {
                        return Err(Box::new(PAError::OutputUnderflowed))
                    }
                }
            }
            Err(e) => return Err(Box::new(e)),
        };

        let buffer_frames = (demod_iq.len() / CHANNELS as usize) as u32;
        let write_frames = std::cmp::min(buffer_frames, out_frames);
        let n_write_samples = write_frames as usize * CHANNELS as usize;

        stream.write(
            write_frames,
            |output| for ix in 0..n_write_samples {
                output[ix] = demod_iq[ix];
            },
        )?;
    }
}

fn init_sdr(
    sdr_index: i32,
    fm_freq: u32,
    bandwidth: u32,
) -> Result<RTLSDRDevice, RTLSDRError> {
    let mut sdr = try!(rtlsdr::open(sdr_index));
    try!(sdr.set_center_freq(fm_freq));
    try!(sdr.set_sample_rate(SDR_SAMPLE_RATE));
    try!(sdr.set_tuner_bandwidth(bandwidth));
    try!(sdr.set_agc_mode(true));
    try!(sdr.reset_buffer());
    Ok(sdr)
}

fn decimate<T: Copy>(signal: Vec<T>, dec_rate: usize) -> Vec<T> {
    let mut ix = 0;
    let new_size = (signal.len() / dec_rate + 1) as usize;
    let mut signal_dec = Vec::<T>::with_capacity(new_size);
    while ix < signal.len() {
        signal_dec.push(signal[ix]);
        ix += dec_rate;
    }
    signal_dec
}

fn read_samples(bytes: Vec<u8>) -> Option<Vec<Complex<f32>>> {
    // First, check that we've been given an even number of bytes. Not sure
    // what to do if we don't get I and Q.
    let bytes_len = bytes.len();
    if bytes_len % 2 != 0 {
        println!("Received an odd number of samples.");
        return None;

    }
    let mut iq_vec: Vec<Complex<f32>> = Vec::with_capacity(bytes_len / 2);

    // Write the values to the complex value and normalize from [0, 255] to
    // [-1, 1].
    for iq in bytes.chunks(2) {
        let iq_cmplx = Complex::new(
            (iq[0] as f32 - 127.0) / 127.0,
            (iq[1] as f32 - 127.0) / 127.0,
        );
        iq_vec.push(iq_cmplx);
    }
    Some(iq_vec)
}

fn demod_fm(
    iq: Vec<Complex<f32>>,
    prev: Complex<f32>,
) -> (Vec<f32>, Complex<f32>) {

    let mut p = prev.clone();
    let mut demod_queue: Vec<f32> = Vec::with_capacity(iq.len());

    let gain = SDR_SAMPLE_RATE as f32 / (2.0 * PI * 75e3);

    for samp in iq.iter() {
        let conj = p.conj() * samp;
        demod_queue.push(conj.arg() * gain);
        p = *samp;
    }
    (demod_queue, p)
}

///
/// Filters a Complex vector.
///
fn filter<T: Copy + Num + Zero>(
    samples: &Vec<Complex<T>>,
    taps: &Vec<T>,
) -> Vec<Complex<T>> {
    // We'll lose taps.len() - 1 samples from the filtering.
    let mut filt_samps: Vec<Complex<T>> =
        Vec::with_capacity(samples.len() - taps.len() + 1);

    for window in samples.windows(taps.len()) {
        let iter = window.iter().zip(taps.iter());
        let filt_samp = iter.fold(Complex::zero(), |acc, (x, y)| acc + *x * *y);
        filt_samps.push(filt_samp);
    }
    filt_samps
}

///
/// Filter a real vector.
///
fn filter_real<T: Copy + Num + Zero>(
    samples: &Vec<T>,
    taps: &Vec<T>,
) -> Vec<T> {
    // We'll lose taps.len() - 1 samples from the filtering.
    let mut filt_samps: Vec<T> =
        Vec::with_capacity(samples.len() - taps.len() + 1);
    for window in samples.windows(taps.len()) {
        let iter = window.iter().zip(taps.iter());
        let filt_samp = iter.fold(T::zero(), |acc, (x, y)| acc + *x * *y);
        filt_samps.push(filt_samp);
    }
    filt_samps
}

///
/// Generates taps for a Hamming window.
///
fn hamming(ntaps: usize) -> Vec<f32> {
    let m: f32 = ntaps as f32 - 1.0;
    let mut taps: Vec<f32> = Vec::with_capacity(ntaps);
    for ix in 0..ntaps {
        taps.push(0.54 - 0.46 * (2.0 * PI * ix as f32 / m).cos());
    }
    taps
}

///
/// Generates filter taps for a sinc filter with a Hamming window.
///
fn windowed_sinc(cutoff_freq: f32, sample_rate: f32, ntaps: usize) -> Vec<f32> {
    let wc: f32 = 2.0 * PI * cutoff_freq / sample_rate;
    let mut taps: Vec<f32> = Vec::with_capacity(ntaps);
    let win_taps: Vec<f32> = hamming(ntaps);
    let ntaps_i: i32 = ntaps as i32;
    let m: i32 = (ntaps_i - 1) / 2;

    for ix in -m..m + 1 {
        let ix_f: f32 = ix as f32;
        if ix == 0 {
            let hi = wc / PI * win_taps[(ix + m) as usize];
            taps.push(hi);
        } else {
            let hi = (ix_f * wc).sin() / (ix_f * PI) *
                win_taps[(ix + m) as usize];
            taps.push(hi);
        }
    }
    taps
}
<commit_msg>Removing plotting code<commit_after>extern crate num;
extern crate portaudio;
extern crate rtlsdr;

use portaudio as pa;
use rtlsdr::{RTLSDRDevice, RTLSDRError};
use num::{Num, Zero};
use num::complex::Complex;
use std::env;
use std::f32::consts::PI;
use std::error::Error;
use pa::error::Error as PAError;
use std::thread;
use std::sync::mpsc::channel;

// Make sure that the buffer size is radix-2, otherwise the read_sync function
// will fail with an error code of -8. I want the buffer sizes between the SDR
// and the audio sizes to match, but since we get two samples per read the
// buffer size needs to be double.
const SDR_BUF_SIZE: usize = 524288;

// The buffer size for the audio sink.
const AUDIO_BUF_SIZE: usize = 262144;

// Most other sample rates fail, but this one works for my particular device.
// I will investigate exactly what's happening here and generate a list of
// possible sample rates (if there are others besides this one).
const SDR_SAMPLE_RATE: u32 = 2400000;

// Here is the sample rate of the output waveform we'll try to use.
const AUDIO_SAMPLE_RATE: f32 = 48000.0;

// The number of channels being used for audio.
const CHANNELS: i32 = 2;

fn main() {
    match run() {
        Ok(()) => (),
        Err(e) => println!("ERROR: {}", e),
    }
}

struct RTLSDR {
    rtlsdr: RTLSDRDevice,
}

unsafe impl Send for RTLSDR {}

fn run() -> Result<(), Box<Error>> {
    let sdr_index: i32 = env::args()
        .nth(1)
        .expect("Please specify the SDR index.")
        .parse::<i32>()?;
    let fm_freq_mhz: f32 = env::args()
        .nth(2)
        .expect("Please specify a center frequency in MHz.")
        .parse::<f32>()?;
    let bandwidth: u32 = env::args()
        .nth(3)
        .expect("Please specify the bandwidth.")
        .parse::<u32>()?;

    // Set up the threading channels for communication between our two threads.
    let (dongle_tx, dongle_rx) = channel();
    let (demod_tx, demod_rx) = channel();

    // This thread only collects data off of the SDR as quickly as possible and
    // sends it out.
    thread::spawn(move || {
        // Initialize the SDR based on the user's input parameters.
        let fm_freq: u32 = (fm_freq_mhz * 1e6) as u32;
        let mut sdr =
            RTLSDR { rtlsdr: init_sdr(sdr_index, fm_freq, bandwidth).unwrap() };

        loop {
            let bytes = sdr.rtlsdr.read_sync(SDR_BUF_SIZE).unwrap();
            let iq_vec = read_samples(bytes).unwrap();
            dongle_tx.send(iq_vec).unwrap();
        }
    });

    // This thread does all of the signal processing required to demodulate
    // the signal and get it to a nice audio stream.
    thread::spawn(move || {
        // Decimating down to 100k samples seems to work the best. I don't
        // really quite get why that is, so I'll need to do more research.
        let dec_rate_filt = 6;
        let dec_rate_audio = 4;

        let taps_filt =
            windowed_sinc(bandwidth as f32, SDR_SAMPLE_RATE as f32, 128);
        let taps_audio = windowed_sinc(
            bandwidth as f32 / dec_rate_filt as f32,
            SDR_SAMPLE_RATE as f32 / dec_rate_filt as f32,
            128,
        );

        let mut prev = Complex::zero();
        loop {
            // First, low_pass and decimate the raw IQ signal.
            let mut iq_vec = dongle_rx.recv().unwrap();
            iq_vec = filter(&iq_vec, &taps_filt);
            iq_vec = decimate(iq_vec, dec_rate_filt);

            // Next, demodulate the signal and filter the signal again. The
            // second filter seems to help with the noise after demodulation.
            let res = demod_fm(iq_vec, prev);
            let mut demod_iq = filter_real(&res.0, &taps_audio);
            demod_iq = decimate(demod_iq, dec_rate_audio);
            prev = res.1;
            demod_tx.send(demod_iq).unwrap();
        }
    });

    // Initialize the PortAudio class. We use a blocking stream since we're
    // doing processing on the signal and can't keep up with non-blocking
    // timing requirements.
    let audio = try!(pa::PortAudio::new());
    let settings = try!(audio.default_output_stream_settings(
        CHANNELS,
        AUDIO_SAMPLE_RATE as f64,
        AUDIO_BUF_SIZE as u32,
    ));
    let mut stream = try!(audio.open_blocking_stream(settings));
    try!(stream.start());

    loop {
        let demod_iq = demod_rx.recv().unwrap();

        let out_frames = match stream.write_available() {
            Ok(available) => {
                match available {
                    pa::StreamAvailable::Frames(frames) => frames as u32,
                    pa::StreamAvailable::InputOverflowed => {
                        return Err(Box::new(PAError::InputOverflowed))
                    }
                    pa::StreamAvailable::OutputUnderflowed => {
                        return Err(Box::new(PAError::OutputUnderflowed))
                    }
                }
            }
            Err(e) => return Err(Box::new(e)),
        };

        let buffer_frames = (demod_iq.len() / CHANNELS as usize) as u32;
        let write_frames = std::cmp::min(buffer_frames, out_frames);
        let n_write_samples = write_frames as usize * CHANNELS as usize;

        stream.write(
            write_frames,
            |output| for ix in 0..n_write_samples {
                output[ix] = demod_iq[ix];
            },
        )?;
    }
}

fn init_sdr(
    sdr_index: i32,
    fm_freq: u32,
    bandwidth: u32,
) -> Result<RTLSDRDevice, RTLSDRError> {
    let mut sdr = try!(rtlsdr::open(sdr_index));
    try!(sdr.set_center_freq(fm_freq));
    try!(sdr.set_sample_rate(SDR_SAMPLE_RATE));
    try!(sdr.set_tuner_bandwidth(bandwidth));
    try!(sdr.set_agc_mode(true));
    try!(sdr.reset_buffer());
    Ok(sdr)
}

fn decimate<T: Copy>(signal: Vec<T>, dec_rate: usize) -> Vec<T> {
    let mut ix = 0;
    let new_size = (signal.len() / dec_rate + 1) as usize;
    let mut signal_dec = Vec::<T>::with_capacity(new_size);
    while ix < signal.len() {
        signal_dec.push(signal[ix]);
        ix += dec_rate;
    }
    signal_dec
}

fn read_samples(bytes: Vec<u8>) -> Option<Vec<Complex<f32>>> {
    // First, check that we've been given an even number of bytes. Not sure
    // what to do if we don't get I and Q.
    let bytes_len = bytes.len();
    if bytes_len % 2 != 0 {
        println!("Received an odd number of samples.");
        return None;

    }
    let mut iq_vec: Vec<Complex<f32>> = Vec::with_capacity(bytes_len / 2);

    // Write the values to the complex value and normalize from [0, 255] to
    // [-1, 1].
    for iq in bytes.chunks(2) {
        let iq_cmplx = Complex::new(
            (iq[0] as f32 - 127.0) / 127.0,
            (iq[1] as f32 - 127.0) / 127.0,
        );
        iq_vec.push(iq_cmplx);
    }
    Some(iq_vec)
}

fn demod_fm(
    iq: Vec<Complex<f32>>,
    prev: Complex<f32>,
) -> (Vec<f32>, Complex<f32>) {

    let mut p = prev.clone();
    let mut demod_queue: Vec<f32> = Vec::with_capacity(iq.len());

    let gain = SDR_SAMPLE_RATE as f32 / (2.0 * PI * 75e3);

    for samp in iq.iter() {
        let conj = p.conj() * samp;
        demod_queue.push(conj.arg() * gain);
        p = *samp;
    }
    (demod_queue, p)
}

///
/// Filters a Complex vector.
///
fn filter<T: Copy + Num + Zero>(
    samples: &Vec<Complex<T>>,
    taps: &Vec<T>,
) -> Vec<Complex<T>> {
    // We'll lose taps.len() - 1 samples from the filtering.
    let mut filt_samps: Vec<Complex<T>> =
        Vec::with_capacity(samples.len() - taps.len() + 1);

    for window in samples.windows(taps.len()) {
        let iter = window.iter().zip(taps.iter());
        let filt_samp = iter.fold(Complex::zero(), |acc, (x, y)| acc + *x * *y);
        filt_samps.push(filt_samp);
    }
    filt_samps
}

///
/// Filter a real vector.
///
fn filter_real<T: Copy + Num + Zero>(
    samples: &Vec<T>,
    taps: &Vec<T>,
) -> Vec<T> {
    // We'll lose taps.len() - 1 samples from the filtering.
    let mut filt_samps: Vec<T> =
        Vec::with_capacity(samples.len() - taps.len() + 1);
    for window in samples.windows(taps.len()) {
        let iter = window.iter().zip(taps.iter());
        let filt_samp = iter.fold(T::zero(), |acc, (x, y)| acc + *x * *y);
        filt_samps.push(filt_samp);
    }
    filt_samps
}

///
/// Generates taps for a Hamming window.
///
fn hamming(ntaps: usize) -> Vec<f32> {
    let m: f32 = ntaps as f32 - 1.0;
    let mut taps: Vec<f32> = Vec::with_capacity(ntaps);
    for ix in 0..ntaps {
        taps.push(0.54 - 0.46 * (2.0 * PI * ix as f32 / m).cos());
    }
    taps
}

///
/// Generates filter taps for a sinc filter with a Hamming window.
///
fn windowed_sinc(cutoff_freq: f32, sample_rate: f32, ntaps: usize) -> Vec<f32> {
    let wc: f32 = 2.0 * PI * cutoff_freq / sample_rate;
    let mut taps: Vec<f32> = Vec::with_capacity(ntaps);
    let win_taps: Vec<f32> = hamming(ntaps);
    let ntaps_i: i32 = ntaps as i32;
    let m: i32 = (ntaps_i - 1) / 2;

    for ix in -m..m + 1 {
        let ix_f: f32 = ix as f32;
        if ix == 0 {
            let hi = wc / PI * win_taps[(ix + m) as usize];
            taps.push(hi);
        } else {
            let hi = (ix_f * wc).sin() / (ix_f * PI) *
                win_taps[(ix + m) as usize];
            taps.push(hi);
        }
    }
    taps
}
<|endoftext|>


<commit_before>use std::string::String;
use std::fs::File;
use std::io::{BufReader, Read};
use std::collections::HashMap;
use std::ptr;

use zip::read::{ZipFile, ZipArchive};
use zip::result::ZipError;
use byteorder::{LittleEndian, ReadBytesExt};
use quick_xml::{XmlReader, Event, AsStr};
use encoding::{Encoding, DecoderTrap};
use encoding::all::UTF_16LE;

use {DataType, ExcelReader, Range, CellErrorType};
use vba::VbaProject;
use utils;
use errors::*;

pub struct Xlsb {
    zip: ZipArchive<File>,
}

impl Xlsb {
    fn iter<'a>(&'a mut self, path: &str) -> Result<RecordIter<'a>> {
        match self.zip.by_name(path) {
            Ok(f) => Ok(RecordIter { r: BufReader::new(f) }),
            Err(ZipError::FileNotFound) => Err(format!("file {} does not exist", path).into()),
            Err(e) => Err(e.into()),
        }
    }
}

impl ExcelReader for Xlsb {

    fn new(f: File) -> Result<Self> {
        Ok(Xlsb { zip: try!(ZipArchive::new(f)) })
    }

    fn has_vba(&mut self) -> bool {
        self.zip.by_name("xl/vbaProject.bin").is_ok()
    }

    fn vba_project(&mut self) -> Result<VbaProject> {
        let mut f = try!(self.zip.by_name("xl/vbaProject.bin"));
        let len = f.size() as usize;
        VbaProject::new(&mut f, len)
    }

    /// MS-XLSB
    fn read_relationships(&mut self) -> Result<HashMap<Vec<u8>, String>> {
        let mut relationships = HashMap::new();
        match self.zip.by_name("xl/_rels/workbook.bin.rels") {
            Ok(f) => {
                let xml = XmlReader::from_reader(BufReader::new(f))
                    .with_check(false)
                    .trim_text(false);

                for res_event in xml {
                    match res_event {
                        Ok(Event::Start(ref e)) if e.name() == b"Relationship" => {
                            let mut id = Vec::new();
                            let mut target = String::new();
                            for a in e.attributes() {
                                match try!(a) {
                                    (b"Id", v) => id.extend_from_slice(v),
                                    (b"Target", v) => target = try!(v.as_str()).to_string(),
                                    _ => (),
                                }
                            }
                            relationships.insert(id, target);
                        }
                        Err(e) => return Err(e.into()),
                        _ => (),
                    }
                }
            },
            Err(ZipError::FileNotFound) => (),
            Err(e) => return Err(e.into()),
        }
        Ok(relationships)
    }

    /// MS-XLSB 2.1.7.45
    fn read_shared_strings(&mut self) -> Result<Vec<String>> {
        let mut iter = try!(self.iter("xl/sharedStrings.bin"));
        let mut buf = vec![0; 1024];

        let _ = try!(iter.fill_next(0x009F, &mut buf)); // BrtBeginSst
        let len = try!((&mut &*buf).read_u32::<LittleEndian>()) as usize;
        let mut strings = Vec::with_capacity(len);

        // BrtSSTItems
        for _ in 0..len {
            let _ = try!(iter.next_skip_blocks(0x0013, &[
                                               (0x0023, Some(0x0024)) // future
                                               ], &mut buf)); // BrtSSTItem
            let flags = buf[0];
            if flags & 0b11000000 != 0 {
                // suppose A and B == 0
                return Err("only regular shared strings are supported".into());
            }
            strings.push(try!(wide_str(&buf[1..])));
        }
        Ok(strings)
    }

    /// MS-XLSB 2.1.7.61
    fn read_sheets_names(&mut self, relationships: &HashMap<Vec<u8>, String>)
        -> Result<HashMap<String, String>>
    {
        let mut iter = try!(self.iter("xl/workbook.bin"));
        let mut buf = vec![0; 1024];

        let _ = try!(iter.fill_next(0x0083, &mut buf)); // BrtBeginBook

        // BrtBeginBundleShs
        let _ = try!(iter.next_skip_blocks(0x008F, &[
                                          (0x0080, None),         // BrtFileVersion
                                          (0x0099, None),         // BrtWbProp
                                          (0x02A4, Some(0x0224)), // File Sharing
                                          (0x0025, Some(0x0026)), // AC blocks
                                          (0x02A5, Some(0x0216)), // Book protection(iso)
                                          (0x0087, Some(0x0088)), // BOOKVIEWS
                                          ], &mut buf));
        let mut sheets = HashMap::new();
        loop {
            match try!(iter.read_type()) {
                0x0090 => return Ok(sheets), // BrtEndBundleShs
                0x009C => (), // BrtBundleSh
                typ => return Err(format!("Expecting end of sheet, got {:x}", typ).into()),
            }
            let len = try!(iter.fill_buffer(&mut buf));
            let rel_len = utils::start_u32(&buf[8..len]);
            if rel_len != 0xFFFFFFFF {
                let rel_len = rel_len as usize * 2;
                let relid = &buf[12..12 + rel_len];
                // converts utf16le to utf8 for HashMap search
                let relid = try!(UTF_16LE.decode(relid, DecoderTrap::Ignore).map_err(|e| e.to_string()));
                let path = format!("xl/{}", relationships[relid.as_bytes()]);
                let name = try!(wide_str(&buf[12 + rel_len..len]));
                sheets.insert(name, path);
            }
        }
    }

    /// MS-XLSB 2.1.7.62
    fn read_worksheet_range(&mut self, path: &str, strings: &[String]) -> Result<Range> {
        let mut iter = try!(self.iter(path));
        let mut buf = vec![0; 1024];

        let _ = try!(iter.fill_next(0x0081, &mut buf)); // BrtBeginSheet

        // BrtWsDim
        let _ = try!(iter.next_skip_blocks(0x0094, &[
                                          (0x0093, None),         // BrtWsProp
                                          ], &mut buf));
        let (position, size) = unchecked_rfx(&buf[..16]);

        if size.0 == 0 || size.1 == 0 {
            return Ok(Range::default());
        }

        // BrtBeginSheetData
        let _ = try!(iter.next_skip_blocks(0x0091, &[
                                          (0x0085, Some(0x0086)), // Views
                                          (0x0025, Some(0x0026)), // AC blocks
                                          (0x01E5, None),         // BrtWsFmtInfo
                                          (0x0186, Some(0x0187)), // Col Infos
                                          ], &mut buf));

        let mut data = vec![DataType::Empty; (size.0 * size.1) as usize];

        // loop through all non empty rows
        loop {
            // BrtRowHdr
            let _ = try!(iter.next_skip_blocks(0x0000, &[
                                              (0x0025, Some(0x0026)), // AC blocks
                                              (0x0023, Some(0x0024)), // future
                                              ], &mut buf));
            let row = utils::start_u32(&buf);

            // get column indexes
            let cols = {
                let span_len = utils::start_u32(&buf[13..]) as usize;
                let mut span_iter = utils::to_u32(&buf[17..]).take(span_len * 2);
                let mut cols = Vec::with_capacity(size.1);
                for _ in 0..span_len {
                    cols.extend(span_iter.next().unwrap()..(span_iter.next().unwrap() + 1));
                }
                cols
            };

            // read all values for the row
            for idx in cols.iter()
                .map(|col| (row - position.0) as usize * size.1 + (*col - position.1) as usize) {
                loop {
                    // read record
                    let typ = try!(iter.read_type());
                    let _ = try!(iter.fill_buffer(&mut buf));

                    match typ {
                        0x0092 => return Err("Expecting cell, got BrtEndSheetData".into()),
                        0x0001 => (), // BrtCellBlank: nothing to do as it is the default value
                        0x0002 => { // BrtCellRk MS-XLSB 2.5.122
                            let d100 = (buf[8] & 1) != 0;
                            let is_int = (buf[8] & 2) != 0;
                            buf[8] &= 0xFC;
                            // TODO: use unchecked if too slow ...
                            data[idx] = if is_int {
                                let v = unsafe {
                                    (ptr::read(&buf[8..12] as *const [u8] as *const i32) >> 2) as i64
                                };
                                DataType::Int( if d100 { v/100 } else { v })
                            } else {
                                let mut v = [0u8; 8];
                                v[4..].copy_from_slice(&buf[8..12]);
                                let v = unsafe {
                                    ptr::read(&v as *const [u8] as *const f64)
                                };
                                DataType::Float( if d100 { v/100.0 } else { v })
                            };
                        },
                        0x0003 => { // BrtCellError
                            data[idx] = DataType::Error(match buf[8] {
                                0x00 => CellErrorType::Null,
                                0x07 => CellErrorType::Div0,
                                0x0F => CellErrorType::Value,
                                0x17 => CellErrorType::Ref,
                                0x1D => CellErrorType::Name,
                                0x24 => CellErrorType::Num,
                                0x2A => CellErrorType::NA,
                                0x2B => CellErrorType::GettingData,
                                c => return Err(format!("Unrecognised cell error code 0x{:x}", c).into()),
                            });
                        },
                        0x0004 => { // BrtCellBool
                            data[idx] = DataType::Bool(buf[8] != 0);
                        },
                        0x0005 => { // BrtCellReal
                            let v = unsafe { ptr::read(&buf[8..16] as *const [u8] as *const f64) };
                            data[idx] = DataType::Float(v);
                        },
                        0x0006 => { // BrtCellSt
                            data[idx] = DataType::String(try!(wide_str(&buf[8..])));
                        },
                        0x0007 => { // BrtCellIsst
                            let isst = utils::start_u32(&buf[8..12]) as usize;
                            data[idx] = DataType::String(strings[isst].clone());
                        },
                        _ => continue, // anything else, ignore and try next
                    }
                    break;
                }
            }

            if (row, *cols.last().unwrap()) == (position.0 + size.0 as u32 -1, position.1 + size.1 as u32 - 1)  {
                return Ok(Range::new(position, size, data))
            }
        }
    }

}

struct RecordIter<'a> {
    r: BufReader<ZipFile<'a>>,
}

impl<'a> RecordIter<'a> {

    fn read_type(&mut self) -> Result<u16> {
        let b = try!(self.r.read_u8());
        let typ = if (b & 0x80) == 0x80 {
            (b & 0x7F) as u16 + (((try!(self.r.read_u8()) & 0x7F) as u16)<<7)
        } else {
            b as u16
        };
        Ok(typ)
    }

    fn fill_buffer(&mut self, buf: &mut Vec<u8>) -> Result<usize> {
        let mut b = try!(self.r.read_u8());
        let mut len = b as usize;
        for i in 0..3 {
            if (b & 0x80) == 0 { break; }
            b = try!(self.r.read_u8());
            len += (b as usize) << (7 * i);
        }
        if buf.len() < len { *buf = vec![0; len]; }

        let _ = try!(self.r.read_exact(&mut buf[..len]));
        Ok(len)
    }

    fn fill_next(&mut self, record_type: u16, buf: &mut Vec<u8>) -> Result<usize> {
        let typ = try!(self.read_type());
        if record_type != typ {
            Err(format!("Unexpected record: expecting 0x{:x}, found 0x{:x}",
                        record_type, typ).into())
        } else {
            self.fill_buffer(buf)
        }
    }

    /// Reads next type, and discard block between `start` and `end`
    fn next_skip_blocks(&mut self, record_type: u16, bounds: &[(u16, Option<u16>)],
                       buf: &mut Vec<u8>) -> Result<usize>
    {
        let mut end = None;
        loop {
            let typ = try!(self.read_type());
            match end {
                Some(e) if e == typ => end = None,
                Some(_) => (),
                None if typ == record_type => {
                    return self.fill_buffer(buf);
                },
                None => match bounds.iter().position(|b| b.0 == typ) {
                    Some(i) => {
                        end = bounds[i].1;
                    },
                    None => return Err(format!("Unexpected record after block: \
                        expecting 0x{:x} found 0x{:x}", record_type, typ).into())
                }
            }
            let _ = try!(self.fill_buffer(buf));
        }
    }

}

fn wide_str(buf: &[u8]) -> Result<String> {
    let len = utils::start_u32(buf);
    let s = &buf[4..4 + len as usize * 2];
    UTF_16LE.decode(s, DecoderTrap::Ignore).map_err(|e| e.to_string().into())
}

fn unchecked_rfx(buf: &[u8]) -> ((u32, u32), (usize, usize)) {
    let mut iter = utils::to_u32(&buf[..16]);
    let rw_first = iter.next().unwrap();
    let rw_last = iter.next().unwrap();
    let col_first = iter.next().unwrap();
    let col_last = iter.next().unwrap();

    ((rw_first, col_first),
     ((rw_last - rw_first + 1) as usize, (col_last - col_first + 1) as usize))
}

// fn nullable_wide_str(buf: &[u8]) -> Result<Option<String>> {
//     let len = utils::start_u32(buf);
//     if len == 0xFFFFFFFF {
//         Ok(None)
//     } else {
//         let s = &buf[4..4 + len as usize * 2];
//         let s = try!(UTF_16LE.decode(s, DecoderTrap::Ignore).map_err(|e| e.to_string()));
//         Ok(Some(s))
//     }
// }
<commit_msg>small cleanings<commit_after>use std::string::String;
use std::fs::File;
use std::io::{BufReader, Read};
use std::collections::HashMap;
use std::ptr;

use zip::read::{ZipFile, ZipArchive};
use zip::result::ZipError;
use byteorder::{LittleEndian, ReadBytesExt};
use quick_xml::{XmlReader, Event, AsStr};
use encoding::{Encoding, DecoderTrap};
use encoding::all::UTF_16LE;

use {DataType, ExcelReader, Range, CellErrorType};
use vba::VbaProject;
use utils;
use errors::*;

pub struct Xlsb {
    zip: ZipArchive<File>,
}

impl Xlsb {
    fn iter<'a>(&'a mut self, path: &str) -> Result<RecordIter<'a>> {
        match self.zip.by_name(path) {
            Ok(f) => Ok(RecordIter { r: BufReader::new(f) }),
            Err(ZipError::FileNotFound) => Err(format!("file {} does not exist", path).into()),
            Err(e) => Err(e.into()),
        }
    }
}

impl ExcelReader for Xlsb {

    fn new(f: File) -> Result<Self> {
        Ok(Xlsb { zip: try!(ZipArchive::new(f)) })
    }

    fn has_vba(&mut self) -> bool {
        self.zip.by_name("xl/vbaProject.bin").is_ok()
    }

    fn vba_project(&mut self) -> Result<VbaProject> {
        let mut f = try!(self.zip.by_name("xl/vbaProject.bin"));
        let len = f.size() as usize;
        VbaProject::new(&mut f, len)
    }

    /// MS-XLSB
    fn read_relationships(&mut self) -> Result<HashMap<Vec<u8>, String>> {
        let mut relationships = HashMap::new();
        match self.zip.by_name("xl/_rels/workbook.bin.rels") {
            Ok(f) => {
                let xml = XmlReader::from_reader(BufReader::new(f))
                    .with_check(false)
                    .trim_text(false);

                for res_event in xml {
                    match res_event {
                        Ok(Event::Start(ref e)) if e.name() == b"Relationship" => {
                            let mut id = Vec::new();
                            let mut target = String::new();
                            for a in e.attributes() {
                                match try!(a) {
                                    (b"Id", v) => id.extend_from_slice(v),
                                    (b"Target", v) => target = try!(v.as_str()).to_string(),
                                    _ => (),
                                }
                            }
                            relationships.insert(id, target);
                        }
                        Err(e) => return Err(e.into()),
                        _ => (),
                    }
                }
            },
            Err(ZipError::FileNotFound) => (),
            Err(e) => return Err(e.into()),
        }
        Ok(relationships)
    }

    /// MS-XLSB 2.1.7.45
    fn read_shared_strings(&mut self) -> Result<Vec<String>> {
        let mut iter = try!(self.iter("xl/sharedStrings.bin"));
        let mut buf = vec![0; 1024];

        let _ = try!(iter.fill_next(0x009F, &mut buf)); // BrtBeginSst
        let len = try!((&mut &*buf).read_u32::<LittleEndian>()) as usize;
        let mut strings = Vec::with_capacity(len);

        // BrtSSTItems
        for _ in 0..len {
            let _ = try!(iter.next_skip_blocks(0x0013, &[
                                               (0x0023, Some(0x0024)) // future
                                               ], &mut buf)); // BrtSSTItem
            let flags = buf[0];
            if flags & 0b11000000 != 0 {
                // suppose A and B == 0
                return Err("only regular shared strings are supported".into());
            }
            strings.push(try!(wide_str(&buf[1..])));
        }
        Ok(strings)
    }

    /// MS-XLSB 2.1.7.61
    fn read_sheets_names(&mut self, relationships: &HashMap<Vec<u8>, String>)
        -> Result<HashMap<String, String>>
    {
        let mut iter = try!(self.iter("xl/workbook.bin"));
        let mut buf = vec![0; 1024];

        let _ = try!(iter.fill_next(0x0083, &mut buf)); // BrtBeginBook

        // BrtBeginBundleShs
        let _ = try!(iter.next_skip_blocks(0x008F, &[
                                          (0x0080, None),         // BrtFileVersion
                                          (0x0099, None),         // BrtWbProp
                                          (0x02A4, Some(0x0224)), // File Sharing
                                          (0x0025, Some(0x0026)), // AC blocks
                                          (0x02A5, Some(0x0216)), // Book protection(iso)
                                          (0x0087, Some(0x0088)), // BOOKVIEWS
                                          ], &mut buf));
        let mut sheets = HashMap::new();
        loop {
            match try!(iter.read_type()) {
                0x0090 => return Ok(sheets), // BrtEndBundleShs
                0x009C => (), // BrtBundleSh
                typ => return Err(format!("Expecting end of sheet, got {:x}", typ).into()),
            }
            let len = try!(iter.fill_buffer(&mut buf));
            let rel_len = utils::start_u32(&buf[8..len]);
            if rel_len != 0xFFFFFFFF {
                let rel_len = rel_len as usize * 2;
                let relid = &buf[12..12 + rel_len];
                // converts utf16le to utf8 for HashMap search
                let relid = try!(UTF_16LE.decode(relid, DecoderTrap::Ignore).map_err(|e| e.to_string()));
                let path = format!("xl/{}", relationships[relid.as_bytes()]);
                let name = try!(wide_str(&buf[12 + rel_len..len]));
                sheets.insert(name, path);
            }
        }
    }

    /// MS-XLSB 2.1.7.62
    fn read_worksheet_range(&mut self, path: &str, strings: &[String]) -> Result<Range> {
        let mut iter = try!(self.iter(path));
        let mut buf = vec![0; 1024];

        let _ = try!(iter.fill_next(0x0081, &mut buf)); // BrtBeginSheet

        // BrtWsDim, skipping BrtWsProp
        let _ = try!(iter.next_skip_blocks(0x0094, &[(0x0093, None)], &mut buf));
        let (position, size) = unchecked_rfx(&buf[..16]);

        if size.0 == 0 || size.1 == 0 {
            return Ok(Range::default());
        }

        // BrtBeginSheetData
        let _ = try!(iter.next_skip_blocks(0x0091, &[
                                          (0x0085, Some(0x0086)), // Views
                                          (0x0025, Some(0x0026)), // AC blocks
                                          (0x01E5, None),         // BrtWsFmtInfo
                                          (0x0186, Some(0x0187)), // Col Infos
                                          ], &mut buf));

        let mut data = vec![DataType::Empty; (size.0 * size.1) as usize];

        // loop through all non empty rows
        loop {
            // BrtRowHdr
            let _ = try!(iter.next_skip_blocks(0x0000, &[
                                              (0x0025, Some(0x0026)), // AC blocks
                                              (0x0023, Some(0x0024)), // future
                                              ], &mut buf));
            let row = utils::start_u32(&buf);

            // get column indexes
            let cols = {
                let span_len = utils::start_u32(&buf[13..]) as usize;
                let mut span_iter = utils::to_u32(&buf[17..]).take(span_len * 2);
                let mut cols = Vec::with_capacity(size.1);
                for _ in 0..span_len {
                    cols.extend(span_iter.next().unwrap()..(span_iter.next().unwrap() + 1));
                }
                cols
            };

            // read all values for the row
            for idx in cols.iter()
                .map(|col| (row - position.0) as usize * size.1 + (*col - position.1) as usize) {
                loop {
                    // read record
                    let typ = try!(iter.read_type());
                    let _ = try!(iter.fill_buffer(&mut buf));

                    match typ {
                        0x0092 => return Err("Expecting cell, got BrtEndSheetData".into()),
                        0x0001 => (), // BrtCellBlank: nothing to do as it is the default value
                        0x0002 => { // BrtCellRk MS-XLSB 2.5.122
                            let d100 = (buf[8] & 1) != 0;
                            let is_int = (buf[8] & 2) != 0;
                            buf[8] &= 0xFC;
                            data[idx] = if is_int {
                                let v = unsafe {
                                    (ptr::read(&buf[8..12] as *const [u8] as *const i32) >> 2) as i64
                                };
                                DataType::Int( if d100 { v/100 } else { v })
                            } else {
                                let mut v = [0u8; 8];
                                v[4..].copy_from_slice(&buf[8..12]);
                                let v = unsafe {
                                    ptr::read(&v as *const [u8] as *const f64)
                                };
                                DataType::Float( if d100 { v/100.0 } else { v })
                            };
                        },
                        0x0003 => { // BrtCellError
                            data[idx] = DataType::Error(match buf[8] {
                                0x00 => CellErrorType::Null,
                                0x07 => CellErrorType::Div0,
                                0x0F => CellErrorType::Value,
                                0x17 => CellErrorType::Ref,
                                0x1D => CellErrorType::Name,
                                0x24 => CellErrorType::Num,
                                0x2A => CellErrorType::NA,
                                0x2B => CellErrorType::GettingData,
                                c => return Err(format!("Unrecognised cell error code 0x{:x}", c).into()),
                            });
                        },
                        0x0004 => { // BrtCellBool
                            data[idx] = DataType::Bool(buf[8] != 0);
                        },
                        0x0005 => { // BrtCellReal
                            let v = unsafe { ptr::read(&buf[8..16] as *const [u8] as *const f64) };
                            data[idx] = DataType::Float(v);
                        },
                        0x0006 => { // BrtCellSt
                            data[idx] = DataType::String(try!(wide_str(&buf[8..])));
                        },
                        0x0007 => { // BrtCellIsst
                            let isst = utils::start_u32(&buf[8..12]) as usize;
                            data[idx] = DataType::String(strings[isst].clone());
                        },
                        _ => continue, // anything else, ignore and try next
                    }

                    // return if the last cell has been set
                    if idx == data.len() - 1 {
                        return Ok(Range::new(position, size, data))
                    } else {
                        break;
                    }
                }
            }
        }
    }

}

struct RecordIter<'a> {
    r: BufReader<ZipFile<'a>>,
}

impl<'a> RecordIter<'a> {

    fn read_type(&mut self) -> Result<u16> {
        let b = try!(self.r.read_u8());
        let typ = if (b & 0x80) == 0x80 {
            (b & 0x7F) as u16 + (((try!(self.r.read_u8()) & 0x7F) as u16)<<7)
        } else {
            b as u16
        };
        Ok(typ)
    }

    fn fill_buffer(&mut self, buf: &mut Vec<u8>) -> Result<usize> {
        let mut b = try!(self.r.read_u8());
        let mut len = b as usize;
        for i in 0..3 {
            if (b & 0x80) == 0 { break; }
            b = try!(self.r.read_u8());
            len += (b as usize) << (7 * i);
        }
        if buf.len() < len { *buf = vec![0; len]; }

        let _ = try!(self.r.read_exact(&mut buf[..len]));
        Ok(len)
    }

    fn fill_next(&mut self, record_type: u16, buf: &mut Vec<u8>) -> Result<usize> {
        let typ = try!(self.read_type());
        if record_type != typ {
            Err(format!("Unexpected record: expecting 0x{:x}, found 0x{:x}",
                        record_type, typ).into())
        } else {
            self.fill_buffer(buf)
        }
    }

    /// Reads next type, and discard block between `start` and `end`
    fn next_skip_blocks(&mut self, record_type: u16, bounds: &[(u16, Option<u16>)],
                       buf: &mut Vec<u8>) -> Result<usize>
    {
        let mut end = None;
        loop {
            let typ = try!(self.read_type());
            match end {
                Some(e) if e == typ => end = None,
                Some(_) => (),
                None if typ == record_type => {
                    return self.fill_buffer(buf);
                },
                None => match bounds.iter().position(|b| b.0 == typ) {
                    Some(i) => {
                        end = bounds[i].1;
                    },
                    None => return Err(format!("Unexpected record after block: \
                        expecting 0x{:x} found 0x{:x}", record_type, typ).into())
                }
            }
            let _ = try!(self.fill_buffer(buf));
        }
    }

}

fn wide_str(buf: &[u8]) -> Result<String> {
    let len = utils::start_u32(buf);
    let s = &buf[4..4 + len as usize * 2];
    UTF_16LE.decode(s, DecoderTrap::Ignore).map_err(|e| e.to_string().into())
}

fn unchecked_rfx(buf: &[u8]) -> ((u32, u32), (usize, usize)) {
    let mut iter = utils::to_u32(&buf[..16]);
    let rw_first = iter.next().unwrap();
    let rw_last = iter.next().unwrap();
    let col_first = iter.next().unwrap();
    let col_last = iter.next().unwrap();

    ((rw_first, col_first),
     ((rw_last - rw_first + 1) as usize, (col_last - col_first + 1) as usize))
}
<|endoftext|>


<commit_before>extern crate mmap;
extern crate time;
extern crate timely;
extern crate getopts;

use timely::progress::timestamp::RootTimestamp;
use timely::progress::nested::Summary::Local;
use timely::example_shared::*;
use timely::example_shared::operators::*;
use timely::communication::*;
use timely::communication::pact::Exchange;
use timely::drain::DrainExt;

mod typedrw;
mod graphmap;
mod sorting;
use graphmap::GraphMMap;
use sorting::{SegmentList, radix_sort_32};

fn main () {

    let source = std::env::args().skip(1).next().unwrap();
    let strategy = std::env::args().skip(2).next().unwrap() == "process";

    // currently need timely's full option set to parse args
    let mut opts = getopts::Options::new();
    opts.optopt("w", "workers", "", "");
    opts.optopt("p", "process", "", "");
    opts.optopt("n", "processes", "", "");
    opts.optopt("h", "hostfile", "", "");

    if let Ok(matches) = opts.parse(std::env::args().skip(3)) {

        let workers: usize = matches.opt_str("w").map(|x| x.parse().unwrap_or(1)).unwrap_or(1);

        timely::initialize(std::env::args().skip(2), move |communicator| {
            pagerank_thread(communicator, &source, workers, strategy);
        });
    }
    else {
        println!("error parsing arguments");
        println!("usage:\tpagerank <source> (worker|process) [timely options]");
    }
}

// returns [src/peers] degrees, (dst, deg) pairs, and a list of [src/peers] endpoints
fn transpose(mut edges: Vec<Vec<(u32, u32)>>, peers: usize, nodes: usize) -> (Vec<u32>, Vec<(u32, u32)>, Vec<u32>)  {

    let mut deg = vec![0; (nodes as usize / peers) + 1];
    for list in &edges {
        for &(s, _) in list {
            deg[s as usize / peers] += 1;
        }
    }

    radix_sort_32(&mut edges, &mut Vec::new(), &|&(_,d)| d);

    let mut rev = Vec::<(u32,u32)>::with_capacity(deg.len());
    let mut trn = Vec::with_capacity(edges.len() * 1024);
    for list in edges {
        for (s,d) in list {
            let len = rev.len();
            if (len == 0) || (rev[len-1].0 < d) {
                rev.push((d, 0u32));
            }

            let len = rev.len();
            rev[len-1].1 += 1;
            trn.push(s / peers as u32);
        }
    }

    return (deg, rev, trn);
}

// pagerank dataflow graph has a set of edges as input, and a binary vertex that for each epoch of
// received edges initiates an iterative subcomputation to compute the pagerank.

fn pagerank_thread<C>(communicator: C, filename: &String, _workers: usize, _process_aggregate: bool)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    let start = time::precise_time_s();

    let nodes = GraphMMap::new(&filename).nodes();

    let mut segments = SegmentList::new(1024); // list of edge segments

    let mut src = vec![];   // holds ranks
    let mut deg = vec![];   // holds source degrees
    let mut rev = vec![];   // holds (dst, deg) pairs
    let mut trn = vec![];   // holds transposed sources

    let mut going = start;

    let mut input = root.subcomputation(|builder| {

        let (input, edges) = builder.new_input::<(u32, u32)>();
        let (cycle, ranks) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let mut ranks = edges.binary_notify(&ranks,
                            Exchange::new(|x: &(u32,u32)| x.0 as u64),
                            Exchange::new(|x: &(u32,f32)| x.0 as u64),
                            format!("pagerank"),
                            vec![RootTimestamp::new(0)],
                            move |input1, input2, output, notificator| {

            // receive incoming edges (should only be iter 0)
            while let Some((_index, data)) = input1.pull() {
                segments.push(data.drain_temp());
            }

            // all inputs received for iter, commence multiplication
            while let Some((iter, _)) = notificator.next() {

                let now = time::now();

                if index == 0 { println!("{}:{}:{}.{} starting iteration {}", now.tm_hour, now.tm_min, now.tm_sec, now.tm_nsec, iter.inner); }

                // if the very first iteration, prepare some stuff.
                // specifically, transpose edges and sort by destination.
                if iter.inner == 0 {
                    let (a, b, c) = transpose(segments.finalize(), peers, nodes);
                    deg = a; rev = b; trn = c;
                    src = vec![0.0f32; deg.len()];
                }

                // record some timings in order to estimate per-iteration times
                if iter.inner == 0  && index == 0 { println!("src: {}, dst: {}, edges: {}", src.len(), rev.len(), trn.len()); }
                if iter.inner == 10 && index == 0 { going = time::precise_time_s(); }
                if iter.inner == 20 && index == 0 { println!("average: {}", (time::precise_time_s() - going) / 10.0 ); }

                // prepare src for transmitting to destinations
                for s in 0..src.len() { src[s] = 0.15 + 0.85 * src[s] / deg[s] as f32; }

                // wander through destinations
                let mut trn_slice = &trn[..];
                let mut session = output.session(&iter);
                for &(dst, deg) in &rev {
                    let mut accum = 0.0;
                    for &s in &trn_slice[..deg as usize] {
                        accum += src[s as usize];
                    }
                    trn_slice = &trn_slice[deg as usize..];
                    session.give((dst, accum));
                }

                // let mut rev_slice = &rev[..];
                // while rev_slice.len() > 0 {
                //     // TODO: session should flush automatically...
                //     let mut session = output.session(&iter);
                //     let next = std::cmp::min(200_000, rev_slice.len());
                //     for &(dst, deg) in &rev_slice[..next] {
                //         let mut accum = 0.0;
                //         for &s in &trn_slice[..deg as usize] {
                //             accum += src[s as usize];
                //         }
                //         trn_slice = &trn_slice[deg as usize..];
                //         session.give((dst, accum));
                //     }
                //     rev_slice = &rev_slice[next..];
                // }

                // clean out src to accumulate into
                for s in &mut src { *s = 0.0; }
            }

            // receive data from workers, accumulate in src
            while let Some((iter, data)) = input2.pull() {
                notificator.notify_at(&iter);
                for x in data.drain_temp() {
                    src[x.0 as usize / peers] += x.1;
                }
            }
        });

        // optionally, do process-local accumulation
        if _process_aggregate {
            let local_base = _workers * (index / _workers);
            let local_index = index % _workers;
            let mut acc = vec![0.0; (nodes / _workers) + 1];   // holds ranks
            ranks = ranks.unary_notify(
                Exchange::new(move |x: &(u32,f32)| (local_base as u64 + (x.0 as u64 % _workers as u64))),
                format!("Aggregation"),
                vec![],
                move |input, output, iterator| {
                    while let Some((iter, data)) = input.pull() {
                        iterator.notify_at(&iter);
                        for x in data.drain_temp() {
                            acc[x.0 as usize / _workers] += x.1;
                        }
                    }

                    while let Some((item, _)) = iterator.next() {
                        output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
                                                 .map(|(u,f)| ((u * _workers + local_index) as u32, f)));

                        for _ in 0..(1 + (nodes/_workers)) { acc.push(0.0); }
                    }
                }
            );
        }

        ranks.connect_loop(cycle);

        input
    });

    // introduce edges into the computation;
    // allow mmaped file to drop
    {
        let graph = GraphMMap::new(&filename);

        let mut edges = Vec::new();
        for node in 0..graph.nodes() {
            if node % peers == index {
                for dst in graph.edges(node) {
                    edges.push((node as u32, *dst as u32));
                }
                if edges.len() > 100_000 {
                    input.send_at(0, edges.drain_temp());
                    root.step();
                }
            }
        }

        input.send_at(0, edges.drain_temp());
    }
    input.close();
    while root.step() { }

    if index == 0 { println!("elapsed: {}", time::precise_time_s() - start); }
}
<commit_msg>hardcoded 128 workers to see...<commit_after>extern crate mmap;
extern crate time;
extern crate timely;
extern crate getopts;

use timely::progress::timestamp::RootTimestamp;
use timely::progress::nested::Summary::Local;
use timely::example_shared::*;
use timely::example_shared::operators::*;
use timely::communication::*;
use timely::communication::pact::Exchange;
use timely::drain::DrainExt;

mod typedrw;
mod graphmap;
mod sorting;
use graphmap::GraphMMap;
use sorting::{SegmentList, radix_sort_32};

fn main () {

    let source = std::env::args().skip(1).next().unwrap();
    let strategy = std::env::args().skip(2).next().unwrap() == "process";

    // currently need timely's full option set to parse args
    let mut opts = getopts::Options::new();
    opts.optopt("w", "workers", "", "");
    opts.optopt("p", "process", "", "");
    opts.optopt("n", "processes", "", "");
    opts.optopt("h", "hostfile", "", "");

    if let Ok(matches) = opts.parse(std::env::args().skip(3)) {

        let workers: usize = matches.opt_str("w").map(|x| x.parse().unwrap_or(1)).unwrap_or(1);

        timely::initialize(std::env::args().skip(2), move |communicator| {
            pagerank_thread(communicator, &source, workers, strategy);
        });
    }
    else {
        println!("error parsing arguments");
        println!("usage:\tpagerank <source> (worker|process) [timely options]");
    }
}

// returns [src/peers] degrees, (dst, deg) pairs, and a list of [src/peers] endpoints
fn transpose(mut edges: Vec<Vec<(u32, u32)>>, peers: usize, nodes: usize) -> (Vec<u32>, Vec<(u32, u32)>, Vec<u32>)  {

    let mut deg = vec![0; (nodes as usize / peers) + 1];
    for list in &edges {
        for &(s, _) in list {
            deg[s as usize / peers] += 1;
        }
    }

    radix_sort_32(&mut edges, &mut Vec::new(), &|&(_,d)| d);

    let mut rev = Vec::<(u32,u32)>::with_capacity(deg.len());
    let mut trn = Vec::with_capacity(edges.len() * 1024);
    for list in edges {
        for (s,d) in list {
            let len = rev.len();
            if (len == 0) || (rev[len-1].0 < d) {
                rev.push((d, 0u32));
            }

            let len = rev.len();
            rev[len-1].1 += 1;
            trn.push(s / peers as u32);
        }
    }

    return (deg, rev, trn);
}

// pagerank dataflow graph has a set of edges as input, and a binary vertex that for each epoch of
// received edges initiates an iterative subcomputation to compute the pagerank.

fn pagerank_thread<C>(communicator: C, filename: &String, _workers: usize, _process_aggregate: bool)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    let start = time::precise_time_s();

    let nodes = GraphMMap::new(&filename).nodes();

    let mut segments = SegmentList::new(1024); // list of edge segments

    let mut src = vec![];   // holds ranks
    let mut deg = vec![];   // holds source degrees
    let mut rev = vec![];   // holds (dst, deg) pairs
    let mut trn = vec![];   // holds transposed sources

    let mut going = start;

    let mut input = root.subcomputation(|builder| {

        let (input, edges) = builder.new_input::<(u32, u32)>();
        let (cycle, ranks) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let mut ranks = edges.binary_notify(&ranks,
                            Exchange::new(|x: &(u32,u32)| x.0 as u64),
                            Exchange::new(|x: &(u32,f32)| x.0 as u64),
                            format!("pagerank"),
                            vec![RootTimestamp::new(0)],
                            move |input1, input2, output, notificator| {

            // receive incoming edges (should only be iter 0)
            while let Some((_index, data)) = input1.pull() {
                segments.push(data.drain_temp());
            }

            // all inputs received for iter, commence multiplication
            while let Some((iter, _)) = notificator.next() {

                let now = time::now();

                if index == 0 { println!("{}:{}:{}.{} starting iteration {}", now.tm_hour, now.tm_min, now.tm_sec, now.tm_nsec, iter.inner); }

                // if the very first iteration, prepare some stuff.
                // specifically, transpose edges and sort by destination.
                if iter.inner == 0 {
                    let (a, b, c) = transpose(segments.finalize(), peers, nodes);
                    deg = a; rev = b; trn = c;
                    src = vec![0.0f32; deg.len()];
                }

                // record some timings in order to estimate per-iteration times
                if iter.inner == 0  && index == 0 { println!("src: {}, dst: {}, edges: {}", src.len(), rev.len(), trn.len()); }
                if iter.inner == 10 && index == 0 { going = time::precise_time_s(); }
                if iter.inner == 20 && index == 0 { println!("average: {}", (time::precise_time_s() - going) / 10.0 ); }

                // prepare src for transmitting to destinations
                for s in 0..src.len() { src[s] = 0.15 + 0.85 * src[s] / deg[s] as f32; }

                // wander through destinations
                let mut trn_slice = &trn[..];
                let mut session = output.session(&iter);
                for &(dst, deg) in &rev {
                    let mut accum = 0.0;
                    for &s in &trn_slice[..deg as usize] {
                        accum += src[s as usize];
                    }
                    trn_slice = &trn_slice[deg as usize..];
                    session.give((dst, accum));
                }

                // let mut rev_slice = &rev[..];
                // while rev_slice.len() > 0 {
                //     // TODO: session should flush automatically...
                //     let mut session = output.session(&iter);
                //     let next = std::cmp::min(200_000, rev_slice.len());
                //     for &(dst, deg) in &rev_slice[..next] {
                //         let mut accum = 0.0;
                //         for &s in &trn_slice[..deg as usize] {
                //             accum += src[s as usize];
                //         }
                //         trn_slice = &trn_slice[deg as usize..];
                //         session.give((dst, accum));
                //     }
                //     rev_slice = &rev_slice[next..];
                // }

                // clean out src to accumulate into
                for s in &mut src { *s = 0.0; }
            }

            // receive data from workers, accumulate in src
            while let Some((iter, data)) = input2.pull() {
                notificator.notify_at(&iter);
                for x in data.drain_temp() {
                    src[x.0 as usize / 128] += x.1;
                }
            }
        });

        // optionally, do process-local accumulation
        if _process_aggregate {
            let local_base = _workers * (index / _workers);
            let local_index = index % _workers;
            let mut acc = vec![0.0; (nodes / _workers) + 1];   // holds ranks
            ranks = ranks.unary_notify(
                Exchange::new(move |x: &(u32,f32)| (local_base as u64 + (x.0 as u64 % _workers as u64))),
                format!("Aggregation"),
                vec![],
                move |input, output, iterator| {
                    while let Some((iter, data)) = input.pull() {
                        iterator.notify_at(&iter);
                        for x in data.drain_temp() {
                            acc[x.0 as usize / _workers] += x.1;
                        }
                    }

                    while let Some((item, _)) = iterator.next() {
                        output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
                                                 .map(|(u,f)| ((u * _workers + local_index) as u32, f)));

                        for _ in 0..(1 + (nodes/_workers)) { acc.push(0.0); }
                    }
                }
            );
        }

        ranks.connect_loop(cycle);

        input
    });

    // introduce edges into the computation;
    // allow mmaped file to drop
    {
        let graph = GraphMMap::new(&filename);

        let mut edges = Vec::new();
        for node in 0..graph.nodes() {
            if node % peers == index {
                for dst in graph.edges(node) {
                    edges.push((node as u32, *dst as u32));
                }
                if edges.len() > 100_000 {
                    input.send_at(0, edges.drain_temp());
                    root.step();
                }
            }
        }

        input.send_at(0, edges.drain_temp());
    }
    input.close();
    while root.step() { }

    if index == 0 { println!("elapsed: {}", time::precise_time_s() - start); }
}
<|endoftext|>


<commit_before>/*
 * Copyright 2020 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

use std::{path::PathBuf, sync::Arc};

use tracing::info;

const VERSION: &str = env!("CARGO_PKG_VERSION");

#[derive(clap::Parser)]
struct Cli {
    #[clap(
        short,
        long,
        env = "QUILKIN_CONFIG",
        default_value = "quilkin.yaml",
        help = "The YAML configuration file."
    )]
    config: PathBuf,
    #[clap(
        short,
        long,
        env,
        help = "Whether Quilkin will report any results to stdout/stderr."
    )]
    quiet: bool,
    #[clap(subcommand)]
    command: Commands,
}

#[derive(clap::Subcommand)]
enum Commands {
    Run,
    GenerateConfigSchema {
        #[clap(
            short,
            long,
            default_value = ".",
            help = "The directory to write configuration files."
        )]
        output_directory: PathBuf,
        #[clap(
            min_values = 1,
            default_value = "all",
            help = "A list of one or more filter IDs to generate or 'all' to generate all available filter schemas."
        )]
        filter_ids: Vec<String>,
    },
    Manage {
        #[clap(subcommand)]
        provider: ProviderCommands,
    },
}

#[derive(clap::Subcommand)]
enum ProviderCommands {
    Agones {
        #[clap(
            short,
            long,
            default_value = "default",
            help = "Namespace under which the proxies run."
        )]
        config_namespace: String,
        #[clap(
            short,
            long,
            default_value = "default",
            help = "Namespace under which the game servers run."
        )]
        gameservers_namespace: String,
    },

    File,
}

#[tokio::main]
async fn main() -> quilkin::Result<()> {
    stable_eyre::install()?;
    let version: std::borrow::Cow<'static, str> = if cfg!(debug_assertions) {
        format!("{VERSION}+debug").into()
    } else {
        VERSION.into()
    };

    let cli = <Cli as clap::Parser>::parse();

    if !cli.quiet {
        let env_filter = tracing_subscriber::EnvFilter::builder()
            .with_default_directive(tracing_subscriber::filter::LevelFilter::INFO.into())
            .from_env_lossy();
        tracing_subscriber::fmt()
            .json()
            .with_env_filter(env_filter)
            .init();
    }

    let config = std::fs::File::open(&cli.config)
        .or_else(|error| {
            if cfg!(unix) {
                std::fs::File::open("/etc/quilkin/quilkin.yaml")
            } else {
                Err(error)
            }
        })
        .map_err(eyre::Error::from)
        .and_then(|file| quilkin::Config::from_reader(file).map_err(From::from))
        .unwrap();

    info!(
        version = &*version,
        commit = quilkin::metadata::build::GIT_COMMIT_HASH,
        "Starting Quilkin"
    );

    match cli.command {
        Commands::Run => quilkin::run(config, vec![]).await,
        Commands::Manage { provider } => {
            let config = Arc::new(config);
            let provider_task = match provider {
                ProviderCommands::Agones {
                    gameservers_namespace,
                    config_namespace,
                } => tokio::spawn(quilkin::config::watch::agones(
                    gameservers_namespace,
                    config_namespace,
                    config.clone(),
                )),
                ProviderCommands::File => {
                    tokio::spawn(quilkin::config::watch::fs(config.clone(), cli.config))
                }
            };

            tokio::select! {
                result = quilkin::manage(config) => result,
                result = provider_task => result?,
            }
        }

        Commands::GenerateConfigSchema {
            output_directory,
            filter_ids,
        } => {
            let set = quilkin::filters::FilterSet::default();
            type SchemaIterator<'r> =
                Box<dyn Iterator<Item = (&'static str, schemars::schema::RootSchema)> + 'r>;

            let schemas = (filter_ids.len() == 1 && filter_ids[0].to_lowercase() == "all")
                .then(|| {
                    Box::new(
                        set.iter()
                            .map(|factory| (factory.name(), factory.config_schema())),
                    ) as SchemaIterator
                })
                .unwrap_or_else(|| {
                    Box::new(filter_ids.iter().filter_map(|id| {
                        let item = set.get(id);

                        if item.is_none() {
                            tracing::error!("{id} not found in filter set.");
                        }

                        item.map(|item| (item.name(), item.config_schema()))
                    })) as SchemaIterator
                });

            for (id, schema) in schemas {
                let mut path = output_directory.join(id);
                path.set_extension("yaml");

                tracing::info!("Writing {id} schema to {}", path.display());

                std::fs::write(path, serde_yaml::to_string(&schema)?)?;
            }

            Ok(())
        }
    }
}
<commit_msg>Only search for config file when required (#560)<commit_after>/*
 * Copyright 2020 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

use std::{path::PathBuf, sync::Arc};

use quilkin::Config;
use tracing::info;

const VERSION: &str = env!("CARGO_PKG_VERSION");

#[derive(clap::Parser)]
struct Cli {
    #[clap(
        short,
        long,
        env = "QUILKIN_CONFIG",
        default_value = "quilkin.yaml",
        help = "The YAML configuration file."
    )]
    config: PathBuf,
    #[clap(
        short,
        long,
        env,
        help = "Whether Quilkin will report any results to stdout/stderr."
    )]
    quiet: bool,
    #[clap(subcommand)]
    command: Commands,
}

#[derive(clap::Subcommand)]
enum Commands {
    Run,
    GenerateConfigSchema {
        #[clap(
            short,
            long,
            default_value = ".",
            help = "The directory to write configuration files."
        )]
        output_directory: PathBuf,
        #[clap(
            min_values = 1,
            default_value = "all",
            help = "A list of one or more filter IDs to generate or 'all' to generate all available filter schemas."
        )]
        filter_ids: Vec<String>,
    },
    Manage {
        #[clap(subcommand)]
        provider: ProviderCommands,
    },
}

#[derive(clap::Subcommand)]
enum ProviderCommands {
    Agones {
        #[clap(
            short,
            long,
            default_value = "default",
            help = "Namespace under which the proxies run."
        )]
        config_namespace: String,
        #[clap(
            short,
            long,
            default_value = "default",
            help = "Namespace under which the game servers run."
        )]
        gameservers_namespace: String,
    },

    File,
}

#[tokio::main]
async fn main() -> quilkin::Result<()> {
    stable_eyre::install()?;
    let version: std::borrow::Cow<'static, str> = if cfg!(debug_assertions) {
        format!("{VERSION}+debug").into()
    } else {
        VERSION.into()
    };

    let cli = <Cli as clap::Parser>::parse();

    if !cli.quiet {
        let env_filter = tracing_subscriber::EnvFilter::builder()
            .with_default_directive(tracing_subscriber::filter::LevelFilter::INFO.into())
            .from_env_lossy();
        tracing_subscriber::fmt()
            .json()
            .with_env_filter(env_filter)
            .init();
    }

    info!(
        version = &*version,
        commit = quilkin::metadata::build::GIT_COMMIT_HASH,
        "Starting Quilkin"
    );

    match cli.command {
        Commands::Run => quilkin::run(read_config(&cli.config), vec![]).await,
        Commands::Manage { provider } => {
            let config = Arc::new(read_config(&cli.config));
            let provider_task = match provider {
                ProviderCommands::Agones {
                    gameservers_namespace,
                    config_namespace,
                } => tokio::spawn(quilkin::config::watch::agones(
                    gameservers_namespace,
                    config_namespace,
                    config.clone(),
                )),
                ProviderCommands::File => {
                    tokio::spawn(quilkin::config::watch::fs(config.clone(), cli.config))
                }
            };

            tokio::select! {
                result = quilkin::manage(config) => result,
                result = provider_task => result?,
            }
        }

        Commands::GenerateConfigSchema {
            output_directory,
            filter_ids,
        } => {
            let set = quilkin::filters::FilterSet::default();
            type SchemaIterator<'r> =
                Box<dyn Iterator<Item = (&'static str, schemars::schema::RootSchema)> + 'r>;

            let schemas = (filter_ids.len() == 1 && filter_ids[0].to_lowercase() == "all")
                .then(|| {
                    Box::new(
                        set.iter()
                            .map(|factory| (factory.name(), factory.config_schema())),
                    ) as SchemaIterator
                })
                .unwrap_or_else(|| {
                    Box::new(filter_ids.iter().filter_map(|id| {
                        let item = set.get(id);

                        if item.is_none() {
                            tracing::error!("{id} not found in filter set.");
                        }

                        item.map(|item| (item.name(), item.config_schema()))
                    })) as SchemaIterator
                });

            for (id, schema) in schemas {
                let mut path = output_directory.join(id);
                path.set_extension("yaml");

                tracing::info!("Writing {id} schema to {}", path.display());

                std::fs::write(path, serde_yaml::to_string(&schema)?)?;
            }

            Ok(())
        }
    }
}

/// Searches for the configuration file, and panics if not found.
fn read_config(path: &PathBuf) -> Config {
    std::fs::File::open(path)
        .or_else(|error| {
            if cfg!(unix) {
                std::fs::File::open("/etc/quilkin/quilkin.yaml")
            } else {
                Err(error)
            }
        })
        .map_err(eyre::Error::from)
        .and_then(|file| quilkin::Config::from_reader(file).map_err(From::from))
        .unwrap()
}
<|endoftext|>


<commit_before>extern crate calamine;
extern crate csv;

use std::env;
use std::path::PathBuf;
use calamine::{Excel, Range, DataType, Result};

fn read_as_excel(sce: PathBuf) {
    let mut xl = Excel::open(&sce).unwrap();

    let range_result = xl.sheet_names()
        .map(|elem| elem[0].to_string())
        .and_then(|name| xl.worksheet_range(&name));
    let range = range_result.unwrap();
    write_range(range).unwrap();
}

fn read_as_csv(sce: PathBuf) {
    let mut rdr = csv::Reader::from_file(&sce).unwrap();
    for row in rdr.records().map(|r| r.unwrap()) {
        let n = row.len() - 1;
        for (i, c) in row.iter().enumerate() {
            print!("{}", c);
            if i != n {
                let _ = print!(",");
            }

        }
        println!("");
    }
}

fn main() {
    // converts first argument into a csv (same name, silently overrides
    // if the file already exists

    let file = env::args()
        .skip(1)
        .next()
        .expect("Please provide an file to show, csv, excel");
    let sce = PathBuf::from(file);
    match sce.extension().and_then(|s| s.to_str()) {
        Some("xlsx") | Some("xlsm") | Some("xlsb") | Some("xls") => read_as_excel(sce),
        Some("csv") | Some("txt") => read_as_csv(sce),
        _ => panic!("Expecting an excel file"),
    }

}

fn write_range(range: Range) -> Result<()> {
    let n = range.get_size().1 - 1;
    for r in range.rows() {
        for (i, c) in r.iter().enumerate() {
            let _ = match *c {
                DataType::Empty => print!(""),
                DataType::String(ref s) => print!("{}", s),
                DataType::Float(ref f) => print!("{}", f),
                DataType::Int(ref i) => print!("{}", i),
                DataType::Error(ref e) => print!("{:?}", e),
                DataType::Bool(ref b) => print!("{}", b),
            };
            if i != n {
                let _ = print!(",");
            }
        }
        let _ = println!("");
    }
    Ok(())
}
<commit_msg>Able to pass multi args<commit_after>extern crate calamine;
extern crate csv;

use std::env;
use std::io::Write;
use std::path::PathBuf;
use calamine::{Excel, Range, DataType, Result};

macro_rules! println_stderr(
    ($($arg:tt)*) => { {
        let r = writeln!(&mut ::std::io::stderr(), $($arg)*);
        r.expect("failed printing to stderr");
    } }
);

fn read_as_excel(sce: PathBuf) {
    let mut xl = Excel::open(&sce).unwrap();

    let range_result = xl.sheet_names()
        .map(|elem| elem[0].to_string())
        .and_then(|name| xl.worksheet_range(&name));
    let range = range_result.unwrap();
    write_range(range).unwrap();
}

fn read_as_csv(sce: PathBuf) {
    let mut rdr = csv::Reader::from_file(&sce).unwrap();
    for row in rdr.records().map(|r| r.unwrap()) {
        let n = row.len() - 1;
        for (i, c) in row.iter().enumerate() {
            print!("{}", c);
            if i != n {
                let _ = print!(",");
            }

        }
        println!("");
    }
}

fn main() {
    // converts first argument into a csv (same name, silently overrides
    // if the file already exists

    for file in env::args().skip(1) {
        let sce = PathBuf::from(&file);
        if !sce.exists() {
            println_stderr!("{}: No such file or directory", file);
            continue;
        }
        match sce.extension().and_then(|s| s.to_str()) {
            Some("xlsx") | Some("xlsm") | Some("xlsb") | Some("xls") => read_as_excel(sce),
            Some("csv") | Some("txt") => read_as_csv(sce),
            _ => panic!("Expecting an excel file"),
        }

    }

}

fn write_range(range: Range) -> Result<()> {
    let n = range.get_size().1 - 1;
    for r in range.rows() {
        for (i, c) in r.iter().enumerate() {
            let _ = match *c {
                DataType::Empty => print!(""),
                DataType::String(ref s) => print!("{}", s),
                DataType::Float(ref f) => print!("{}", f),
                DataType::Int(ref i) => print!("{}", i),
                DataType::Error(ref e) => print!("{:?}", e),
                DataType::Bool(ref b) => print!("{}", b),
            };
            if i != n {
                let _ = print!(",");
            }
        }
        let _ = println!("");
    }
    Ok(())
}
<|endoftext|>


<commit_before>#![crate_name="magog"]
#![feature(old_io, old_path, unicode)]

extern crate image;
extern crate "calx_util" as util;

#[macro_use]
extern crate "calx_backend" as backend;

extern crate world;
extern crate time;

use backend::{Canvas};

use gamestate::GameState;
use titlestate::TitleState;

pub static SCREEN_W: u32 = 640;
pub static SCREEN_H: u32 = 360;

pub mod drawable;
pub mod tilecache;
pub mod viewutil;
pub mod worldview;
mod gamestate;
mod titlestate;
mod sprite;
mod msg_queue;
mod console;

pub trait State {
    fn process(&mut self, event: backend::Event) -> Option<Transition>;
}

pub enum Transition {
    Game(Option<u32>),
    Title,
    Exit,
}

pub fn version() -> String {
    let next_release = "0.1.0";
    let git_hash = include_str!("git_hash.inc");
    // Set is_release to true for one commit to make a release.
    let is_release = false;

    if is_release {
        format!("{}", next_release)
    } else {
        format!("{}-alpha+g{}", next_release, git_hash)
    }
}

pub fn compiler_version() -> String {
    include_str!("../rustc_version.txt").to_string()
}

pub fn screenshot(ctx: &mut Canvas) {
    use time;
    use std::old_io::File;
    use image;

    let shot = ctx.screenshot();
    let mut file = File::create(&Path::new(
            format!("/tmp/shot-{}.png", time::precise_time_s() as u64)))
            .unwrap();
    let _ = image::ImageRgb8(shot).save(&mut file, image::PNG);
}

pub fn main() {
    println!("Magog v{}", version());
    println!("{}", compiler_version());
    let mut canvas = backend::CanvasBuilder::new()
        .set_size(SCREEN_W, SCREEN_H)
        .set_frame_interval(0.030f64);
    tilecache::init(&mut canvas);
    let mut state: Box<State> = Box::new(TitleState::new());

    for evt in canvas.run() {
        match state.process(evt) {
            Some(Transition::Title) => { state = Box::new(TitleState::new()); }
            Some(Transition::Game(seed)) => {
                state = Box::new(GameState::new(seed)); }
            Some(Transition::Exit) => { break; }
            _ => ()
        }
    }
}
<commit_msg>More new-IO-y and less file-clobbery screenshot routine<commit_after>#![crate_name="magog"]
#![feature(path_ext, old_path)]

extern crate image;
extern crate "calx_util" as util;

#[macro_use]
extern crate "calx_backend" as backend;

extern crate world;
extern crate time;

use backend::{Canvas};

use gamestate::GameState;
use titlestate::TitleState;

pub static SCREEN_W: u32 = 640;
pub static SCREEN_H: u32 = 360;

pub mod drawable;
pub mod tilecache;
pub mod viewutil;
pub mod worldview;
mod gamestate;
mod titlestate;
mod sprite;
mod msg_queue;
mod console;

pub trait State {
    fn process(&mut self, event: backend::Event) -> Option<Transition>;
}

pub enum Transition {
    Game(Option<u32>),
    Title,
    Exit,
}

pub fn version() -> String {
    let next_release = "0.1.0";
    let git_hash = include_str!("git_hash.inc");
    // Set is_release to true for one commit to make a release.
    let is_release = false;

    if is_release {
        format!("{}", next_release)
    } else {
        format!("{}-alpha+g{}", next_release, git_hash)
    }
}

pub fn compiler_version() -> String {
    include_str!("../rustc_version.txt").to_string()
}

pub fn screenshot(ctx: &mut Canvas) {
    use time;
    use std::path;
    use std::fs::PathExt;
    use image;

    let timestamp = time::precise_time_s() as u64;

    // Create screenshot filenames by concatenating the current timestamp in
    // seconds with a running number from 00 to 99. 100 shots per second
    // should be good enough.

    // Default if we fail to generate any of the 100 candidates for this
    // second, just overwrite with the "xx" prefix then.
    let mut filename = format!("magog-{}{}.png", timestamp, "xx");

    // Run through candidates for this second.
    for i in 0..100 {
        let test_filename = format!("magog-{}{:02}.png", timestamp, i);
        if !path::Path::new(&test_filename).exists() {
            filename = test_filename;
            break;
        }
    }

    let shot = ctx.screenshot();
    let _ = image::save_buffer(&Path::new(filename), shot.as_slice(), shot.width(), shot.height(), image::ColorType::RGB(8));
}

pub fn main() {
    println!("Magog v{}", version());
    println!("{}", compiler_version());
    let mut canvas = backend::CanvasBuilder::new()
        .set_size(SCREEN_W, SCREEN_H)
        .set_frame_interval(0.030f64);
    tilecache::init(&mut canvas);
    let mut state: Box<State> = Box::new(TitleState::new());

    for evt in canvas.run() {
        match state.process(evt) {
            Some(Transition::Title) => { state = Box::new(TitleState::new()); }
            Some(Transition::Game(seed)) => {
                state = Box::new(GameState::new(seed)); }
            Some(Transition::Exit) => { break; }
            _ => ()
        }
    }
}
<|endoftext|>


<commit_before>#![crate_id = "risp"]
#![crate_type = "bin"]

//! A Lisp interpreter.

extern crate libc;

use libc::c_char;
use std::c_str::CString;

#[link(name = "linenoise")]
extern {
    fn linenoise(p: *c_char) -> *c_char;
    fn linenoiseHistoryAdd(l: *c_char);
}

/// Attempts to read input from a user using linenoise. Returns an option,
/// Some(StrBuf) for success, or None if EOF (^D) is entered.
pub fn rust_linenoise(prompt: &str) -> Option<StrBuf>
{
    if prompt.len() == 0 {
        return None
    }

    let c_prompt = prompt.to_c_str();

    c_prompt.with_ref(|c_buf| {
        unsafe {
            let ret_str = CString::new(linenoise(c_buf), true);
            if ret_str.is_not_null() {
                ret_str.as_str().map(|ret_str| ret_str.to_strbuf())
            } else {
                None
            }
        }
    })
}

/// Adds a string to a history buffer.
pub fn rust_add_history(line: &str) {
    if line.len() == 0 {
        return
    }

    let c_line = line.to_c_str();
    c_line.with_ref(|c_line| {
        unsafe {
            linenoiseHistoryAdd(c_line);
        }
    });
}

fn main()
{
    loop {
        let expr = match rust_linenoise(">>> ") {
            Some(val)   => { val.to_str() },
            None    => { continue }
        };
        rust_add_history(expr);

        match expr.trim() {
            "(exit)" | "exit" | ",q"    => { break },
            _   => { println!("{}", expr); }
        }
    }
}
<commit_msg>Multiple line input is now acceptable<commit_after>#![crate_id = "risp"]
#![crate_type = "bin"]

//! A Lisp interpreter.

extern crate libc;

use libc::c_char;
use std::c_str::CString;

//pub mod read;

#[link(name = "linenoise")]
extern {
    fn linenoise(p: *c_char) -> *c_char;
    fn linenoiseHistoryAdd(l: *c_char);
}

/// Attempts to read input from a user using linenoise. Returns an option,
/// Some(StrBuf) for success, or None if EOF (^D) is entered.
pub fn rust_linenoise(prompt: &str) -> Option<StrBuf> {
    if prompt.len() == 0 {
        return None
    }

    let c_prompt = prompt.to_c_str();

    c_prompt.with_ref(|c_buf| {
        unsafe {
            let ret_str = CString::new(linenoise(c_buf), true);
            if ret_str.is_not_null() {
                ret_str.as_str().map(|ret_str| ret_str.to_strbuf())
            } else {
                None
            }
        }
    })
}

/// Adds a string to a history buffer.
pub fn rust_add_history(line: &str) {
    if line.len() == 0 {
        return
    }

    let c_line = line.to_c_str();
    c_line.with_ref(|c_line| {
        unsafe {
            linenoiseHistoryAdd(c_line);
        }
    });
}

/// Make sure there are equal amounts of parens and prevent malformed
/// expressions such as ``(+ 2 2)(+ 2 2)"
pub fn count_parens(expr: &str) -> (uint, uint, bool) {
    let (mut lparens, mut rparens) = (0, 0);
    for c in expr.chars() {
        if c == '(' {
            if lparens == rparens && lparens != 0 {
                return (lparens, rparens, false)
            } else {
                lparens += 1
            }
        } else if c == ')' {
            if rparens >= lparens && rparens != 0 {
                return (lparens, rparens, false)
            } else {
                rparens += 1
            }
        }
    }

    (lparens, rparens, true)
}

fn main() {
    loop {
        let mut expr = match rust_linenoise(">>> ") {
            Some(val)   => { val.to_str() },
            None    => { continue }
        };

        let (mut lparens, mut rparens): (uint, uint);
        //A valid expression can have no parens if and only if it is a
        //constant or variable, eg 2, x, 'd' or "hello, world!"
        let mut okay_expr = true;

        if expr.trim().starts_with("(") {
            loop {
                let (lparenstmp, rparenstmp, okay_exprtmp) = count_parens(expr.as_slice());
                lparens = lparenstmp;
                rparens = rparenstmp;
                okay_expr = okay_exprtmp;
                if okay_expr == false {
                    break
                }

                if lparens == rparens {
                    okay_expr = true;
                    break
                }

                 let expr_part = match rust_linenoise(" ") {
                    Some(val)   => { val.to_str() },
                    None    => { continue }
                };

                expr = expr + "\n" + expr_part;
            }
        }

        if okay_expr == false {
            println!("Bad expression");
            continue
        }

        rust_add_history(expr);

        match expr.trim() {
            "(exit)" | "exit" | ",q"    => { break },
            _   => { println!("{}", expr); }
        }
    }
}
<|endoftext|>


<commit_before>const NUM_DIRECTIONS: usize = 4;

enum Direction {
  North,
  South,
  East,
  West,
}

struct Room<'a> {
  exits: [Option<&'a Room<'a>>; NUM_DIRECTIONS],
  name: &'a str,
  description: &'a str,
}

impl<'a> Room<'a> {
  pub fn new() -> Self {
    Self {
      exits: [None, None, None, None],
      name: "",
      description: "",
    }
  }

  pub fn get_exit(self, d: Direction) -> Option<&'a Room<'a>> {
    self.exits[d as usize]
  }

  pub fn set_exit(&mut self, d: Direction, room: &'a Room<'a>) -> &mut Self {
    self.exits[d as usize] = Some(room);
    self
  }
}

fn main() {
  let mut b = Room::new();
  let mut a = Room::new();

  a.set_exit(Direction::North, &mut b);
}
<commit_msg>Add Map class<commit_after>const NUM_DIRECTIONS: usize = 4;
const NUM_ROOMS: usize = 2;

#[derive(Debug, Copy, Clone)]
enum RoomId {
  Hallway,
  AudienceChamber,
}

#[derive(Debug)]
struct Map<'a> {
  rooms: [Room<'a>; NUM_ROOMS]
}

impl<'a> Map<'a> {
  pub fn new() -> Self {
    Self { rooms: [Room::new(); NUM_ROOMS] }
  }

  pub fn connect(&mut self, from: RoomId, d: Direction, to: RoomId) {
    self.rooms[from as usize].set_exit(d, to);
    self.rooms[to as usize].set_exit(d.opposite(), from);
  }
}

#[derive(Debug, Copy, Clone)]
enum Direction {
  North,
  South,
  East,
  West,
}

impl Direction {
  pub fn opposite(self) -> Self {
    match self {
      Direction::North => Direction::South,
      Direction::South => Direction::North,
      Direction::East => Direction::West,
      Direction::West => Direction::East,
    }
  }
}

#[derive(Debug, Copy, Clone)]
struct Room<'a> {
  exits: [Option<RoomId>; NUM_DIRECTIONS],
  name: &'a str,
  description: &'a str,
}

impl<'a> Room<'a> {
  pub fn new() -> Self {
    Self {
      exits: [None; NUM_DIRECTIONS],
      name: "",
      description: "",
    }
  }

  pub fn get_exit(self, d: Direction) -> Option<RoomId> {
    self.exits[d as usize]
  }

  pub fn set_exit(&mut self, d: Direction, r: RoomId) -> &mut Self {
    self.exits[d as usize] = Some(r);
    self
  }
}

fn main() {
  let mut map = Map::new();

  map.connect(RoomId::Hallway, Direction::South, RoomId::AudienceChamber);

  println!("Map is {:?}", map);
}
<|endoftext|>