<commit_before>// Claxon -- A FLAC decoding library in Rust
// Copyright 2017 Ruud van Asseldonk
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// A copy of the License has been included in the root of the repository.

// This file contains a minimal example of using Claxon with mp4parse and Hound
// to decode a flac stream inside an MP4 container to a wav file.

extern crate claxon;
extern crate hound;
extern crate mp4parse;

use std::env;
use std::path::Path;
use std::fs::File;
use std::io;
use std::io::Seek;

use claxon::metadata::read_metadata_block;
use claxon::metadata::StreamInfo;
use hound::{WavSpec, WavWriter};
use mp4parse::CodecType;

fn decode_file(fname: &Path) {
    // Create a file to read the mp4 structure from. We will later need to seek
    // in this file to get to the parts that contain FLAC data.
    let file = File::open(fname).expect("failed to open mp4 file");
    let mut bufread = io::BufReader::new(file);

    // Parse the mp4 metadata.
    let mut context = mp4parse::MediaContext::new();
    mp4parse::read_mp4(&mut bufread, &mut context).expect("failed to decode mp4");

    // An MP4 file contains one or more tracks. A track can contain a single
    // FLAC stream. Iterate over those.
    for track in &context.tracks {
        if track.codec_type == CodecType::FLAC {
            let streaminfo = get_streaminfo(track).expect("missing streaminfo");

            // Build a wav writer to write the decoded track to a wav file.
            let spec = WavSpec {
                channels: streaminfo.channels as u16,
                sample_rate: streaminfo.sample_rate,
                bits_per_sample: streaminfo.bits_per_sample as u16,
                sample_format: hound::SampleFormat::Int,
            };

            let fname_wav = fname.with_extension("wav");
            let opt_wav_writer = WavWriter::create(fname_wav, spec);
            let mut wav_writer = opt_wav_writer.expect("failed to create wav file");

            // The data in an MP4 file is split into "chunks", for which we can
            // get the file offset where the chunk starts. Every chunk contains
            // one or more "samples", which is one FLAC frame. These frames are
            // all stored adjacently in the chunk, so we can read them with a
            // fingle frame reader.

            // The "stco" in mp4parse's Track stands for "Chunk Offset Box",
            // "stsc" stands for "Sample to Chunk Box", which actually tells per
            // chunk how many samples it contains, not per sample in which chunk
            // it is.
            let chunk_offsets = &track.stco.as_ref().expect("missing chunk offset box").offsets;
            let chunk_samples = &track.stsc.as_ref().expect("missing sample to chunk box").samples;

            let mut ck_off_iter = chunk_offsets.iter().enumerate();

            for cs in chunk_samples {
                // The "sample" contains the index of the chunk that contains
                // the sample and subsequent samples. Locate the file offset of
                // that chunk in the "Chunk Offset Box", then seek to there. The
                // first_chunk field is a 1-based chunk index, not 0-based.
                let i_off = ck_off_iter.find(|&(i, _off)| 1 + i as u32 == cs.first_chunk);
                let seek_to = i_off.expect("failed to locate chunk offset for mp4 sample").1;
                bufread.seek(io::SeekFrom::Start(*seek_to)).expect("failed to seek to chunk");

                // Decode all the "samples" (FLAC frames) in the chunk.
                bufread = decode_frames(bufread, &streaminfo, cs.samples_per_chunk, &mut wav_writer);
            }

            // Stop iterating over tracks; if there are more FLAC tracks, we
            // would overwrite the previously written output. (This could be
            // avoided by picking a unique file name for every track, or by
            // asking the user which track to decode.)
            break
        }
    }
}

/// Decode the metadata blocks until the streaminfo block is found.
fn get_streaminfo(track: &mp4parse::Track) -> Option<StreamInfo> {
    use mp4parse::{AudioCodecSpecific, SampleEntry};
    use claxon::metadata::MetadataBlock;

    let audio_entry = match &track.data {
        &Some(SampleEntry::Audio(ref ae)) => ae,
        _ => panic!("expected to find audio entry in FLAC track"),
    };

    let flac_box = match &audio_entry.codec_specific {
        &AudioCodecSpecific::FLACSpecificBox(ref fb) => fb,
        _ => return None,
    };

    for raw_block in &flac_box.blocks {
        let len = raw_block.data.len() as u32;
        let mut cursor = io::Cursor::new(&raw_block.data);
        let result = read_metadata_block(&mut cursor, raw_block.block_type, len);
        match result.expect("failed to decode metadata block") {
            MetadataBlock::StreamInfo(si) => return Some(si),
            _ => {}
        }
    }

    None
}

/// Decode a number of FLAC frames. Takes an input `io::Read` and returns it.
fn decode_frames<R, W>(input: R,
                       streaminfo: &StreamInfo,
                       num_frames: u32,
                       wav_writer: &mut WavWriter<W>)
                       -> R
where R: io::Read, W: io::Write + io::Seek {
    // The simplest way to read frames now, is unfortunately to double buffer.
    // (The input `R` is a buffered reader.) This might be avoided by not
    // wrapping the original file in an `io::BufReader`, but in a Claxon
    // `BufferedReader`. It would have to implement `io::Read` then.
    let buffered_reader = claxon::input::BufferedReader::new(input);
    let mut frame_reader = claxon::frame::FrameReader::new(buffered_reader);
    let mut buffer = Vec::with_capacity(streaminfo.max_block_size as usize);

    for _ in 0..num_frames {
        // TODO There should be a read_next method too that does not tolerate
        // EOF.
        let result = frame_reader.read_next_or_eof(buffer);
        let block = result.expect("failed to decode frame").expect("unexpected EOF");

        // TODO: Here we assume that we are decoding a stereo stream, which
        // is wrong, but very convenient, as there is no interleaved sample
        // iterator for `Block`. One should be added.
        for (sl, sr) in block.stereo_samples() {
            wav_writer.write_sample(sl).expect("failed to write wav file");
            wav_writer.write_sample(sr).expect("failed to write wav file");
        }

        buffer = block.into_buffer();
    }

    // Strip off the frame reader and buffered reader to get back the original
    // reader.
    frame_reader.into_inner().into_inner()
}

fn main() {
    for fname in env::args().skip(1) {
        decode_file(&Path::new(&fname));
    }
}
<commit_msg>Avoid rightward drift in mp4 example<commit_after>// Claxon -- A FLAC decoding library in Rust
// Copyright 2017 Ruud van Asseldonk
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// A copy of the License has been included in the root of the repository.

// This file contains a minimal example of using Claxon with mp4parse and Hound
// to decode a flac stream inside an MP4 (.mp4 or .m4a) container to a wav file.

extern crate claxon;
extern crate hound;
extern crate mp4parse;

use std::env;
use std::fs::File;
use std::io::Seek;
use std::io;
use std::path::Path;

use claxon::metadata::read_metadata_block;
use claxon::metadata::StreamInfo;
use hound::{WavSpec, WavWriter};
use mp4parse::CodecType;

fn decode_file(fname: &Path) {
    // Create a file to read the mp4 structure from. We will later need to seek
    // in this file to get to the parts that contain FLAC data.
    let file = File::open(fname).expect("failed to open mp4 file");
    let mut bufread = io::BufReader::new(file);

    // Parse the mp4 metadata.
    let mut context = mp4parse::MediaContext::new();
    mp4parse::read_mp4(&mut bufread, &mut context).expect("failed to decode mp4");

    // An MP4 file contains one or more tracks. A track can contain a single
    // FLAC stream. Iterate over those.
    for track in &context.tracks {
        if track.codec_type != CodecType::FLAC {
            continue
        }

        let streaminfo = get_streaminfo(track).expect("missing streaminfo");

        // Build a wav writer to write the decoded track to a wav file.
        let spec = WavSpec {
            channels: streaminfo.channels as u16,
            sample_rate: streaminfo.sample_rate,
            bits_per_sample: streaminfo.bits_per_sample as u16,
            sample_format: hound::SampleFormat::Int,
        };

        let fname_wav = fname.with_extension("wav");
        let opt_wav_writer = WavWriter::create(fname_wav, spec);
        let mut wav_writer = opt_wav_writer.expect("failed to create wav file");

        // The data in an MP4 file is split into "chunks", for which we can get
        // the file offset where the chunk starts. Every chunk contains one or
        // more "samples", which is one FLAC frame. These frames are all stored
        // adjacently in the chunk, so we can read them with a fingle frame
        // reader.

        // The "stco" in mp4parse's Track stands for "Chunk Offset Box", "stsc"
        // stands for "Sample to Chunk Box", which actually tells per chunk how
        // many samples it contains, not per sample in which chunk it is.
        let chunk_offsets = &track.stco.as_ref().expect("missing chunk offset box").offsets;
        let chunk_samples = &track.stsc.as_ref().expect("missing sample to chunk box").samples;

        let mut ck_off_iter = chunk_offsets.iter().enumerate();

        for cs in chunk_samples {
            // The "sample" contains the index of the chunk that contains the
            // sample and subsequent samples. Locate the file offset of that
            // chunk in the "Chunk Offset Box", then seek to there. The
            // first_chunk field is a 1-based chunk index, not 0-based.
            let i_off = ck_off_iter.find(|&(i, _off)| 1 + i as u32 == cs.first_chunk);
            let seek_to = i_off.expect("failed to locate chunk offset for mp4 sample").1;
            bufread.seek(io::SeekFrom::Start(*seek_to)).expect("failed to seek to chunk");

            // Decode all the "samples" (FLAC frames) in the chunk.
            bufread = decode_frames(bufread, &streaminfo, cs.samples_per_chunk, &mut wav_writer);
        }

        // Stop iterating over tracks; if there are more FLAC tracks, we would
        // overwrite the previously written output. (This could be avoided by
        // picking a unique file name for every track, or by asking the user
        // which track to decode.)
        break
    }
}

/// Decode the metadata blocks until the streaminfo block is found.
fn get_streaminfo(track: &mp4parse::Track) -> Option<StreamInfo> {
    use mp4parse::{AudioCodecSpecific, SampleEntry};
    use claxon::metadata::MetadataBlock;

    let audio_entry = match &track.data {
        &Some(SampleEntry::Audio(ref ae)) => ae,
        _ => panic!("expected to find audio entry in FLAC track"),
    };

    let flac_box = match &audio_entry.codec_specific {
        &AudioCodecSpecific::FLACSpecificBox(ref fb) => fb,
        _ => return None,
    };

    for raw_block in &flac_box.blocks {
        let len = raw_block.data.len() as u32;
        let mut cursor = io::Cursor::new(&raw_block.data);
        let result = read_metadata_block(&mut cursor, raw_block.block_type, len);
        match result.expect("failed to decode metadata block") {
            MetadataBlock::StreamInfo(si) => return Some(si),
            _ => {}
        }
    }

    None
}

/// Decode a number of FLAC frames. Takes an input `io::Read` and returns it.
fn decode_frames<R, W>(input: R,
                       streaminfo: &StreamInfo,
                       num_frames: u32,
                       wav_writer: &mut WavWriter<W>)
                       -> R
where R: io::Read, W: io::Write + io::Seek {
    // The simplest way to read frames now, is unfortunately to double buffer.
    // (The input `R` is a buffered reader.) This might be avoided by not
    // wrapping the original file in an `io::BufReader`, but in a Claxon
    // `BufferedReader`. It would have to implement `io::Read` then.
    let buffered_reader = claxon::input::BufferedReader::new(input);
    let mut frame_reader = claxon::frame::FrameReader::new(buffered_reader);
    let mut buffer = Vec::with_capacity(streaminfo.max_block_size as usize);

    for _ in 0..num_frames {
        // TODO There should be a read_next method too that does not tolerate
        // EOF.
        let result = frame_reader.read_next_or_eof(buffer);
        let block = result.expect("failed to decode frame").expect("unexpected EOF");

        // TODO: Here we assume that we are decoding a stereo stream, which
        // is wrong, but very convenient, as there is no interleaved sample
        // iterator for `Block`. One should be added.
        for (sl, sr) in block.stereo_samples() {
            wav_writer.write_sample(sl).expect("failed to write wav file");
            wav_writer.write_sample(sr).expect("failed to write wav file");
        }

        buffer = block.into_buffer();
    }

    // Strip off the frame reader and buffered reader to get back the original
    // reader.
    frame_reader.into_inner().into_inner()
}

fn main() {
    for fname in env::args().skip(1) {
        decode_file(&Path::new(&fname));
    }
}
<|endoftext|>


<commit_before>use std::fs;
use std::path::{Path, PathBuf};

use heck::SnekCase;
use itertools::Itertools;
use quote::{ToTokens, Tokens, Ident};

use cfg::CargonautsConfig;

use ast::*;

pub fn formats(routes: &Routes, cfg: &CargonautsConfig) -> Tokens {
    let root = cfg.templates();
    let method_refs = &MethodRef::build(routes).into_iter().group_by(|m| m.format);
    let builders = method_refs.into_iter().map(|(format, methods)| {
        let format = Ident::new(format);

        let templates = methods.filter_map(|method| {
            method.template(&root).map(|path| (path, method))
        }).map(|(path, method)| {
            let r = method.resource;
            let m = method.method;
            let key = match method.relation {
                Some(rel)   => quote!(::cargonauts::formats::TemplateKey::new_rel(#r, #rel, #m)),
                None        => quote!(::cargonauts::formats::TemplateKey::new(#r, #m)),
            };
            Template { key, path }
        });

        quote!({
            <#format as ::cargonauts::formats::BuildFormat>::build(&[
                #(#templates,)*
            ])?
        })
    });
    quote!({
        let mut formats = ::cargonauts::routing::FormatLender::default();
        #(formats.store(#builders);)*
        Ok(formats)
    })
}

#[derive(Copy, Clone)]
struct MethodRef<'a> {
    format: &'a str,
    method: &'a str,
    resource: &'a str,
    relation: Option<&'a str>,
}

impl<'a> MethodRef<'a> {
    fn build(routes: &'a Routes) -> Vec<MethodRef<'a>> {
        let mut vec = vec![];
        routes.visit_resources(&mut vec, |vec, resource| {
            for member in &resource.members {
                match *member {
                    ResourceMember::Method(ref m, _)    => {
                        for method in &m.methods {
                            vec.push(MethodRef {
                                format: &m.format[..],
                                method: &method[..],
                                resource: &resource.header.ty[..],
                                relation: None,
                            });
                        }
                    }
                    ResourceMember::Relation(ref rel, _) => {
                        for member in &rel.members {
                            match *member {
                                RelationMember::Method(ref m, _) => {
                                    for method in &m.methods {
                                        vec.push(MethodRef {
                                            format: &m.format[..],
                                            method: &method[..],
                                            resource: &resource.header.ty[..],
                                            relation: Some(&rel.rel[..]),
                                        });
                                    }
                                }
                            }
                        }
                    }
                }
            }
        });
        vec
    }

    fn template(self, root: &Path) -> Option<PathBuf> {
        let mut root = root.join(self.resource.to_snek_case());
        if let Some(rel) = self.relation { root.push(rel.to_snek_case()); }
        let read_dir = if let Some(rd) = fs::read_dir(root).ok() { rd } else { return None };
        for entry in read_dir {
            let entry = entry.unwrap();
            if entry.file_name().to_str().unwrap().starts_with(&self.method.to_snek_case()) {
                return Some(entry.path())
            }
        }
        None
    }
}

struct Template {
    path: PathBuf,
    key: Tokens,
}

impl ToTokens for Template {
    fn to_tokens(&self, tokens: &mut Tokens) {
        let path = self.path.to_string_lossy();
        let key = &self.key;
        tokens.append(quote! {
            ::cargonauts::formats::Template {
                key: #key,
                template: include_str!(#path),
            }
        })
    }
}
<commit_msg>Check template filenames correctly.<commit_after>use std::fs;
use std::path::{Path, PathBuf};

use heck::SnekCase;
use itertools::Itertools;
use quote::{ToTokens, Tokens, Ident};

use cfg::CargonautsConfig;

use ast::*;

pub fn formats(routes: &Routes, cfg: &CargonautsConfig) -> Tokens {
    let root = cfg.templates();
    let method_refs = &MethodRef::build(routes).into_iter().group_by(|m| m.format);
    let builders = method_refs.into_iter().map(|(format, methods)| {
        let format = Ident::new(format);

        let templates = methods.filter_map(|method| {
            method.template(&root).map(|path| (path, method))
        }).map(|(path, method)| {
            let r = method.resource;
            let m = method.method;
            let key = match method.relation {
                Some(rel)   => quote!(::cargonauts::formats::TemplateKey::new_rel(#r, #rel, #m)),
                None        => quote!(::cargonauts::formats::TemplateKey::new(#r, #m)),
            };
            Template { key, path }
        });

        quote!({
            <#format as ::cargonauts::formats::BuildFormat>::build(&[
                #(#templates,)*
            ])?
        })
    });
    quote!({
        let mut formats = ::cargonauts::routing::FormatLender::default();
        #(formats.store(#builders);)*
        Ok(formats)
    })
}

#[derive(Copy, Clone)]
struct MethodRef<'a> {
    format: &'a str,
    method: &'a str,
    resource: &'a str,
    relation: Option<&'a str>,
}

impl<'a> MethodRef<'a> {
    fn build(routes: &'a Routes) -> Vec<MethodRef<'a>> {
        let mut vec = vec![];
        routes.visit_resources(&mut vec, |vec, resource| {
            for member in &resource.members {
                match *member {
                    ResourceMember::Method(ref m, _)    => {
                        for method in &m.methods {
                            vec.push(MethodRef {
                                format: &m.format[..],
                                method: &method[..],
                                resource: &resource.header.ty[..],
                                relation: None,
                            });
                        }
                    }
                    ResourceMember::Relation(ref rel, _) => {
                        for member in &rel.members {
                            match *member {
                                RelationMember::Method(ref m, _) => {
                                    for method in &m.methods {
                                        vec.push(MethodRef {
                                            format: &m.format[..],
                                            method: &method[..],
                                            resource: &resource.header.ty[..],
                                            relation: Some(&rel.rel[..]),
                                        });
                                    }
                                }
                            }
                        }
                    }
                }
            }
        });
        vec
    }

    fn template(self, root: &Path) -> Option<PathBuf> {
        let mut root = root.join(self.resource.to_snek_case());
        if let Some(rel) = self.relation { root.push(rel.to_snek_case()); }
        let read_dir = if let Some(rd) = fs::read_dir(root).ok() { rd } else { return None };
        for entry in read_dir {
            let entry = entry.unwrap();
            let filename = entry.file_name();
            if self.method.to_snek_case() == filename.to_str().unwrap().split('.').next().unwrap() {
                return Some(entry.path())
            }
        }
        None
    }
}

struct Template {
    path: PathBuf,
    key: Tokens,
}

impl ToTokens for Template {
    fn to_tokens(&self, tokens: &mut Tokens) {
        let path = self.path.to_string_lossy();
        let key = &self.key;
        tokens.append(quote! {
            ::cargonauts::formats::Template {
                key: #key,
                template: include_str!(#path),
            }
        })
    }
}
<|endoftext|>


<commit_before>extern crate rand;
extern crate timely;
extern crate differential_dataflow;

#[macro_use]
extern crate abomonation_derive;
extern crate abomonation;
#[macro_use]
extern crate serde_derive;
extern crate serde;

use rand::{Rng, SeedableRng, StdRng};

use timely::dataflow::*;
use timely::dataflow::operators::probe::Handle;

use differential_dataflow::input::Input;
use differential_dataflow::Collection;
use differential_dataflow::operators::*;
use differential_dataflow::lattice::Lattice;

type Node = u32;
type Edge = (Node, Node);

#[derive(Abomonation, Copy, Ord, PartialOrd, Eq, PartialEq, Debug, Clone, Serialize, Deserialize, Hash)]
pub struct MinSum {
    value: u32,
}

use std::ops::{AddAssign, Mul};
use differential_dataflow::difference::Semigroup;

impl<'a> AddAssign<&'a Self> for MinSum {
    fn add_assign(&mut self, rhs: &'a Self) {
        self.value = std::cmp::min(self.value, rhs.value);
    }
}

impl Mul<Self> for MinSum {
    type Output = Self;
    fn mul(self, rhs: Self) -> Self {
        MinSum { value: self.value + rhs.value }
    }
}

impl Semigroup for MinSum {
    fn is_zero(&self) -> bool { false }
}

fn main() {

    let nodes: u32 = std::env::args().nth(1).unwrap().parse().unwrap();
    let edges: u32 = std::env::args().nth(2).unwrap().parse().unwrap();
    let weight: u32 = std::env::args().nth(3).unwrap().parse().unwrap();
    let batch: u32 = std::env::args().nth(4).unwrap().parse().unwrap();
    let rounds: u32 = std::env::args().nth(5).unwrap().parse().unwrap();
    let inspect: bool = std::env::args().nth(6).unwrap() == "inspect";

    // define a new computational scope, in which to run BFS
    timely::execute_from_args(std::env::args(), move |worker| {

        let timer = ::std::time::Instant::now();

        // define BFS dataflow; return handles to roots and edges inputs
        let mut probe = Handle::new();
        let (mut roots, mut graph) = worker.dataflow(|scope| {

            let (root_input, roots) = scope.new_collection();
            let (edge_input, graph) = scope.new_collection();

            let mut result = bfs(&graph, &roots);

            if !inspect {
                result = result.filter(|_| false);
            }

            result.count()
                  .map(|(_,l)| l)
                  .consolidate()
                  .inspect(|x| println!("\t{:?}", x))
                  .probe_with(&mut probe);

            (root_input, edge_input)
        });

        let seed: &[_] = &[1, 2, 3, 4];
        let mut rng1: StdRng = SeedableRng::from_seed(seed);    // rng for edge additions

        roots.update_at(0, Default::default(), MinSum { value: 0 });
        roots.close();

        println!("performing BFS on {} nodes, {} edges:", nodes, edges);

        if worker.index() == 0 {
            for _ in 0 .. edges {
                graph.update_at(
                    (rng1.gen_range(0, nodes), rng1.gen_range(0, nodes)),
                    Default::default(),
                    MinSum { value: rng1.gen_range(0, weight) },
                );
            }
        }

        println!("{:?}\tloaded", timer.elapsed());

        graph.advance_to(1);
        graph.flush();
        worker.step_while(|| probe.less_than(graph.time()));

        println!("{:?}\tstable", timer.elapsed());

        for round in 0 .. rounds {
            for element in 0 .. batch {
                if worker.index() == 0 {
                    graph.update_at(
                        (rng1.gen_range(0, nodes), rng1.gen_range(0, nodes)),
                        round+1,
                        MinSum { value: rng1.gen_range(0, weight) },
                    );
                }
                graph.advance_to(2 + round * batch + element);
            }
            graph.flush();

            let timer2 = ::std::time::Instant::now();
            worker.step_while(|| probe.less_than(&graph.time()));

            if worker.index() == 0 {
                let elapsed = timer2.elapsed();
                println!("{:?}\t{:?}:\t{}", timer.elapsed(), round, elapsed.as_secs() * 1000000000 + (elapsed.subsec_nanos() as u64));
            }
        }
        println!("finished; elapsed: {:?}", timer.elapsed());
    }).unwrap();
}

// returns pairs (n, s) indicating node n can be reached from a root in s steps.
fn bfs<G: Scope>(edges: &Collection<G, Edge, MinSum>, roots: &Collection<G, Node, MinSum>) -> Collection<G, Node, MinSum>
where G::Timestamp: Lattice+Ord {

    // repeatedly update minimal distances each node can be reached from each root
    roots.scope().iterative::<u32,_,_>(|scope| {

        use differential_dataflow::operators::iterate::SemigroupVariable;
        use differential_dataflow::operators::reduce::ReduceCore;
        use differential_dataflow::trace::implementations::ord::OrdKeySpine as DefaultKeyTrace;

        use timely::order::Product;
        let variable = SemigroupVariable::new(scope, Product::new(Default::default(), 1));

        let edges = edges.enter(scope);
        let roots = roots.enter(scope);

        let result =
        variable
            .map(|n| (n,()))
            .join_map(&edges, |_k,&(),d| *d)
            .concat(&roots)
            .map(|x| (x,()))
            .reduce_core::<_,DefaultKeyTrace<_,_,_>>(|_key, input, output, updates| {
                if output.is_empty() || input[0].1 < output[0].1 {
                    updates.push(((), input[0].1));
                }
            })
            .as_collection(|k,()| *k);

        variable.set(&result);
        result.leave()
     })
}<commit_msg>error in timestamps<commit_after>extern crate rand;
extern crate timely;
extern crate differential_dataflow;

#[macro_use]
extern crate abomonation_derive;
extern crate abomonation;
#[macro_use]
extern crate serde_derive;
extern crate serde;

use rand::{Rng, SeedableRng, StdRng};

use timely::dataflow::*;
use timely::dataflow::operators::probe::Handle;

use differential_dataflow::input::Input;
use differential_dataflow::Collection;
use differential_dataflow::operators::*;
use differential_dataflow::lattice::Lattice;

type Node = u32;
type Edge = (Node, Node);

#[derive(Abomonation, Copy, Ord, PartialOrd, Eq, PartialEq, Debug, Clone, Serialize, Deserialize, Hash)]
pub struct MinSum {
    value: u32,
}

use std::ops::{AddAssign, Mul};
use differential_dataflow::difference::Semigroup;

impl<'a> AddAssign<&'a Self> for MinSum {
    fn add_assign(&mut self, rhs: &'a Self) {
        self.value = std::cmp::min(self.value, rhs.value);
    }
}

impl Mul<Self> for MinSum {
    type Output = Self;
    fn mul(self, rhs: Self) -> Self {
        MinSum { value: self.value + rhs.value }
    }
}

impl Semigroup for MinSum {
    fn is_zero(&self) -> bool { false }
}

fn main() {

    let nodes: u32 = std::env::args().nth(1).unwrap().parse().unwrap();
    let edges: u32 = std::env::args().nth(2).unwrap().parse().unwrap();
    let weight: u32 = std::env::args().nth(3).unwrap().parse().unwrap();
    let batch: u32 = std::env::args().nth(4).unwrap().parse().unwrap();
    let rounds: u32 = std::env::args().nth(5).unwrap().parse().unwrap();
    let inspect: bool = std::env::args().nth(6).unwrap() == "inspect";

    // define a new computational scope, in which to run BFS
    timely::execute_from_args(std::env::args(), move |worker| {

        let timer = ::std::time::Instant::now();

        // define BFS dataflow; return handles to roots and edges inputs
        let mut probe = Handle::new();
        let (mut roots, mut graph) = worker.dataflow(|scope| {

            let (root_input, roots) = scope.new_collection();
            let (edge_input, graph) = scope.new_collection();

            let mut result = bfs(&graph, &roots);

            if !inspect {
                result = result.filter(|_| false);
            }

            result.count()
                  .map(|(_,l)| l)
                  .consolidate()
                  .inspect(|x| println!("\t{:?}", x))
                  .probe_with(&mut probe);

            (root_input, edge_input)
        });

        let seed: &[_] = &[1, 2, 3, 4];
        let mut rng1: StdRng = SeedableRng::from_seed(seed);    // rng for edge additions

        roots.update_at(0, Default::default(), MinSum { value: 0 });
        roots.close();

        println!("performing BFS on {} nodes, {} edges:", nodes, edges);

        if worker.index() == 0 {
            for _ in 0 .. edges {
                graph.update_at(
                    (rng1.gen_range(0, nodes), rng1.gen_range(0, nodes)),
                    Default::default(),
                    MinSum { value: rng1.gen_range(0, weight) },
                );
            }
        }

        println!("{:?}\tloaded", timer.elapsed());

        graph.advance_to(1);
        graph.flush();
        worker.step_while(|| probe.less_than(graph.time()));

        println!("{:?}\tstable", timer.elapsed());

        for round in 0 .. rounds {
            for element in 0 .. batch {
                if worker.index() == 0 {
                    graph.update_at(
                        (rng1.gen_range(0, nodes), rng1.gen_range(0, nodes)),
                        1 + round * batch + element,
                        MinSum { value: rng1.gen_range(0, weight) },
                    );
                }
                graph.advance_to(2 + round * batch + element);
            }
            graph.flush();

            let timer2 = ::std::time::Instant::now();
            worker.step_while(|| probe.less_than(&graph.time()));

            if worker.index() == 0 {
                let elapsed = timer2.elapsed();
                println!("{:?}\t{:?}:\t{}", timer.elapsed(), round, elapsed.as_secs() * 1000000000 + (elapsed.subsec_nanos() as u64));
            }
        }
        println!("finished; elapsed: {:?}", timer.elapsed());
    }).unwrap();
}

// returns pairs (n, s) indicating node n can be reached from a root in s steps.
fn bfs<G: Scope>(edges: &Collection<G, Edge, MinSum>, roots: &Collection<G, Node, MinSum>) -> Collection<G, Node, MinSum>
where G::Timestamp: Lattice+Ord {

    // repeatedly update minimal distances each node can be reached from each root
    roots.scope().iterative::<u32,_,_>(|scope| {

        use differential_dataflow::operators::iterate::SemigroupVariable;
        use differential_dataflow::operators::reduce::ReduceCore;
        use differential_dataflow::trace::implementations::ord::OrdKeySpine as DefaultKeyTrace;

        use timely::order::Product;
        let variable = SemigroupVariable::new(scope, Product::new(Default::default(), 1));

        let edges = edges.enter(scope);
        let roots = roots.enter(scope);

        let result =
        variable
            .map(|n| (n,()))
            .join_map(&edges, |_k,&(),d| *d)
            .concat(&roots)
            .map(|x| (x,()))
            .reduce_core::<_,DefaultKeyTrace<_,_,_>>(|_key, input, output, updates| {
                if output.is_empty() || input[0].1 < output[0].1 {
                    updates.push(((), input[0].1));
                }
            })
            .as_collection(|k,()| *k);

        variable.set(&result);
        result.leave()
     })
}<|endoftext|>


<commit_before>use mio::net::UdpSocket;
use mio::{Events, Interests, Poll, Token};
use std::io;

const IN: Token = Token(0);

fn main() -> io::Result<()> {
    // Set up a new socket on port 9000 to listen on.
    let socket = UdpSocket::bind("0.0.0.0:9000".parse().unwrap())?;
    // Initialize poller.
    let mut poll = Poll::new()?;
    // Register our socket with the token IN (defined above) and an interest
    // in being `READABLE`.
    poll.registry().register(&socket, IN, Interests::READABLE)?;

    // Prepare a buffer for the number of events we can handle at a time.
    // Someone might wat to echo really fast so lets give it some size.
    let mut events = Events::with_capacity(1024);
    // Initialize a buffer for the UDP datagram
    let mut buf = [0; 65535];
    // Main loop
    loop {
        // Poll if we have events waiting for us on the socket.
        poll.poll(&mut events, None)?;
        // If we do iterate throuigh them
        for event in events.iter() {
            // Validate the token we registered our socket with,
            // in this example it will only ever be one but we
            // make sure it's valid none the less.
            match event.token() {
                IN => loop {
                    // In this loop we receive from the socket as long as we
                    // can read data
                    match socket.recv_from(&mut buf) {
                        Ok((n, from_addr)) => {
                            // Send the data right back from where it came from.
                            socket.send_to(&buf[..n], from_addr)?;
                        }
                        Err(e) => {
                            // If we failed to receive data we have two cases
                            if e.kind() == io::ErrorKind::WouldBlock {
                                // If the reason was `WouldBlock` we know
                                // our socket has no more data to give so
                                // we can return to the poll to wait politely.
                                break;
                            } else {
                                // If it was any other kind of error, something
                                // went wrong and we terminate with an error.
                                return Err(e);
                            }
                        }
                    }
                },
                // We only have IN as a token, so this should never ever be hit
                _ => unreachable!(),
            }
        }
    }
}
<commit_msg>Cleanup UDP example<commit_after>use std::io;

use mio::net::UdpSocket;
use mio::{Events, Interests, Poll, Token};

// A token to allow us to identify which event is for the `UdpSocket`.
const UDP_SOCKET: Token = Token(0);

fn main() -> io::Result<()> {
    env_logger::init();

    // Create a poll instance.
    let mut poll = Poll::new()?;
    // Create storage for events. Since we will only register a single socket, a
    // capacity of 1 will do.
    let mut events = Events::with_capacity(1);

    // Setup the UDP socket.
    let addr = "127.0.0.1:9000".parse().unwrap();
    let socket = UdpSocket::bind(addr)?;

    // Register our socket with the token defined above and an interest in being
    // `READABLE`.
    poll.registry()
        .register(&socket, UDP_SOCKET, Interests::READABLE)?;

    println!("You can connect to the server using `nc`:");
    println!(" $ nc -u 127.0.0.1 9000");
    println!("Anything you type will be echoed back to you.");

    // Initialize a buffer for the UDP packet. We use the maximum size of a UDP
    // packet, which is the maximum value of 16 a bit integer.
    let mut buf = [0; 1 << 16];

    // Our event loop.
    loop {
        // Poll to check if we have events waiting for us.
        poll.poll(&mut events, None)?;

        // Process each event.
        for event in events.iter() {
            // Validate the token we registered our socket with,
            // in this example it will only ever be one but we
            // make sure it's valid none the less.
            match event.token() {
                UDP_SOCKET => loop {
                    // In this loop we receive all packets queued for the socket.
                    match socket.recv_from(&mut buf) {
                        Ok((packet_size, source_address)) => {
                            // Echo the data.
                            socket.send_to(&buf[..packet_size], source_address)?;
                        }
                        Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
                            // If we get a `WouldBlock` error we know our socket
                            // has no more packets queued, so we can return to
                            // polling and wait for some more.
                            break;
                        }
                        Err(e) => {
                            // If it was any other kind of error, something went
                            // wrong and we terminate with an error.
                            return Err(e);
                        }
                    }
                },
                // This should never happen as we only registered our
                // `UdpSocket` using the `UDP_SOCKET` token.
                _ => {}
            }
        }
    }
}
<|endoftext|>


<commit_before>use decoders::ljpeg::*;
use decoders::basics::*;
use itertools::Itertools;

pub fn decode_ljpeg_2components(ljpeg: &LjpegDecompressor, out: &mut [u16], x: usize, stripwidth:usize, width: usize, height: usize) -> Result<(),String> {
  if ljpeg.sof.width*2 < width || ljpeg.sof.height < height {
    return Err(format!("ljpeg: trying to decode {}x{} into {}x{}",
                       ljpeg.sof.width*2, ljpeg.sof.height,
                       width, height).to_string())
  }
  let ref htable1 = ljpeg.dhts[ljpeg.sof.components[0].dc_tbl_num];
  let ref htable2 = ljpeg.dhts[ljpeg.sof.components[1].dc_tbl_num];
  let mut pump = BitPumpJPEG::new(ljpeg.buffer);

  let base_prediction = 1 << (ljpeg.sof.precision - ljpeg.point_transform -1);
  out[x]   = (base_prediction + try!(htable1.huff_decode(&mut pump))) as u16;
  out[x+1] = (base_prediction + try!(htable2.huff_decode(&mut pump))) as u16;
  let skip_x = ljpeg.sof.width - width/2;

  for row in 0..height {
    let startcol = if row == 0 {x+2} else {x};
    for col in (startcol..(width+x)).step(2) {
      let (p1,p2) = if col == x {
        // At start of line predictor starts with start of previous line
        (out[(row-1)*stripwidth+x],out[(row-1)*stripwidth+1+x])
      } else {
        // All other cases use the two previous pixels in the same line
        (out[row*stripwidth+col-2], out[row*stripwidth+col-1])
      };

      let diff1 = try!(htable1.huff_decode(&mut pump));
      let diff2 = try!(htable2.huff_decode(&mut pump));
      out[row*stripwidth+col] = ((p1 as i32) + diff1) as u16;
      out[row*stripwidth+col+1] = ((p2 as i32) + diff2) as u16;
    }
    // Skip extra encoded differences if the ljpeg frame is wider than the output
    for _ in 0..skip_x {
      try!(htable1.huff_decode(&mut pump));
      try!(htable2.huff_decode(&mut pump));
    }
  }

  Ok(())
}

pub fn decode_ljpeg_4components(ljpeg: &LjpegDecompressor, out: &mut [u16], width: usize, height: usize) -> Result<(),String> {
  if ljpeg.sof.width*4 < width || ljpeg.sof.height < height {
    return Err(format!("ljpeg: trying to decode {}x{} into {}x{}",
                       ljpeg.sof.width*4, ljpeg.sof.height,
                       width, height).to_string())
  }
  let ref htable1 = ljpeg.dhts[ljpeg.sof.components[0].dc_tbl_num];
  let ref htable2 = ljpeg.dhts[ljpeg.sof.components[1].dc_tbl_num];
  let ref htable3 = ljpeg.dhts[ljpeg.sof.components[2].dc_tbl_num];
  let ref htable4 = ljpeg.dhts[ljpeg.sof.components[3].dc_tbl_num];
  let mut pump = BitPumpJPEG::new(ljpeg.buffer);

  let base_prediction = 1 << (ljpeg.sof.precision - ljpeg.point_transform -1);
  out[0] = (base_prediction + try!(htable1.huff_decode(&mut pump))) as u16;
  out[1] = (base_prediction + try!(htable2.huff_decode(&mut pump))) as u16;
  out[2] = (base_prediction + try!(htable3.huff_decode(&mut pump))) as u16;
  out[3] = (base_prediction + try!(htable4.huff_decode(&mut pump))) as u16;
  let skip_x = ljpeg.sof.width - width/4;

  for row in 0..height {
    let startcol = if row == 0 {4} else {0};
    for col in (startcol..width).step(4) {
      let pos = if col == 0 {
        // At start of line predictor starts with start of previous line
        (row-1)*width
      } else {
        // All other cases use the two previous pixels in the same line
        row*width+col-4
      };

      let (p1,p2,p3,p4) = (out[pos],out[pos+1],out[pos+2],out[pos+3]);

      let diff1 = try!(htable1.huff_decode(&mut pump));
      let diff2 = try!(htable2.huff_decode(&mut pump));
      let diff3 = try!(htable3.huff_decode(&mut pump));
      let diff4 = try!(htable4.huff_decode(&mut pump));
      out[row*width+col] = ((p1 as i32) + diff1) as u16;
      out[row*width+col+1] = ((p2 as i32) + diff2) as u16;
      out[row*width+col+2] = ((p3 as i32) + diff3) as u16;
      out[row*width+col+3] = ((p4 as i32) + diff4) as u16;
    }
    // Skip extra encoded differences if the ljpeg frame is wider than the output
    for _ in 0..skip_x {
      try!(htable1.huff_decode(&mut pump));
      try!(htable2.huff_decode(&mut pump));
      try!(htable3.huff_decode(&mut pump));
      try!(htable4.huff_decode(&mut pump));
    }
  }

  Ok(())
}

fn set_yuv_420(out: &mut [u16], row: usize, col: usize, width: usize, y1: i32, y2: i32, y3: i32, y4: i32, cb: i32, cr: i32) {
  let pix1 = row*width+col;
  let pix2 = pix1+3;
  let pix3 = (row+1)*width+col;
  let pix4 = pix3+3;

  out[pix1+0] = y1 as u16;
  out[pix1+1] = cb as u16;
  out[pix1+2] = cr as u16;
  out[pix2+0] = y2 as u16;
  out[pix2+1] = cb as u16;
  out[pix2+2] = cr as u16;
  out[pix3+0] = y3 as u16;
  out[pix3+1] = cb as u16;
  out[pix3+2] = cr as u16;
  out[pix4+0] = y4 as u16;
  out[pix4+1] = cb as u16;
  out[pix4+2] = cr as u16;
}

pub fn decode_ljpeg_420(ljpeg: &LjpegDecompressor, out: &mut [u16], width: usize, height: usize) -> Result<(),String> {
  if ljpeg.sof.width*3 < width || ljpeg.sof.height < height {
    return Err(format!("ljpeg: trying to decode {}x{} into {}x{}",
                       ljpeg.sof.width*3, ljpeg.sof.height,
                       width, height).to_string())
  }
  let ref htable1 = ljpeg.dhts[ljpeg.sof.components[0].dc_tbl_num];
  let ref htable2 = ljpeg.dhts[ljpeg.sof.components[1].dc_tbl_num];
  let ref htable3 = ljpeg.dhts[ljpeg.sof.components[2].dc_tbl_num];
  let mut pump = BitPumpJPEG::new(ljpeg.buffer);

  let base_prediction = 1 << (ljpeg.sof.precision - ljpeg.point_transform -1);
  let y1 = base_prediction + try!(htable1.huff_decode(&mut pump));
  let y2 = y1 + try!(htable1.huff_decode(&mut pump));
  let y3 = y2 + try!(htable1.huff_decode(&mut pump));
  let y4 = y3 + try!(htable1.huff_decode(&mut pump));
  let cb = base_prediction + try!(htable2.huff_decode(&mut pump));
  let cr = base_prediction + try!(htable3.huff_decode(&mut pump));
  set_yuv_420(out, 0, 0, width, y1, y2, y3, y4, cb, cr);

  for row in (0..height).step(2) {
    let startcol = if row == 0 {6} else {0};
    for col in (startcol..width).step(6) {
      let pos = if col == 0 {
        // At start of line predictor starts with first pixel of start of previous line
        (row-2)*width
      } else {
        // All other cases use the last pixel in the same two lines
        (row+1)*width+col-3
      };
      let (py,pcb,pcr) = (out[pos],out[pos+1],out[pos+2]);

      let y1 = (py  as i32) + try!(htable1.huff_decode(&mut pump));
      let y2 = (y1  as i32) + try!(htable1.huff_decode(&mut pump));
      let y3 = (y2  as i32) + try!(htable1.huff_decode(&mut pump));
      let y4 = (y3  as i32) + try!(htable1.huff_decode(&mut pump));
      let cb = (pcb as i32) + try!(htable2.huff_decode(&mut pump));
      let cr = (pcr as i32) + try!(htable3.huff_decode(&mut pump));
      set_yuv_420(out, row, col, width, y1, y2, y3, y4, cb, cr);
    }
  }

  Ok(())
}

fn set_yuv_422(out: &mut [u16], row: usize, col: usize, width: usize, y1: i32, y2: i32, cb: i32, cr: i32) {
  let pix1 = row*width+col;
  let pix2 = pix1+3;

  out[pix1+0] = y1 as u16;
  out[pix1+1] = cb as u16;
  out[pix1+2] = cr as u16;
  out[pix2+0] = y2 as u16;
  out[pix2+1] = cb as u16;
  out[pix2+2] = cr as u16;
}

pub fn decode_ljpeg_422(ljpeg: &LjpegDecompressor, out: &mut [u16], width: usize, height: usize) -> Result<(),String> {
  if ljpeg.sof.width*3 < width || ljpeg.sof.height < height {
    return Err(format!("ljpeg: trying to decode {}x{} into {}x{}",
                       ljpeg.sof.width*3, ljpeg.sof.height,
                       width, height).to_string())
  }
  let ref htable1 = ljpeg.dhts[ljpeg.sof.components[0].dc_tbl_num];
  let ref htable2 = ljpeg.dhts[ljpeg.sof.components[1].dc_tbl_num];
  let ref htable3 = ljpeg.dhts[ljpeg.sof.components[2].dc_tbl_num];
  let mut pump = BitPumpJPEG::new(ljpeg.buffer);

  let base_prediction = 1 << (ljpeg.sof.precision - ljpeg.point_transform -1);
  let y1 = base_prediction + try!(htable1.huff_decode(&mut pump));
  let y2 = y1 + try!(htable1.huff_decode(&mut pump));
  let cb = base_prediction + try!(htable2.huff_decode(&mut pump));
  let cr = base_prediction + try!(htable3.huff_decode(&mut pump));
  set_yuv_422(out, 0, 0, width, y1, y2, cb, cr);

  for row in 0..height {
    let startcol = if row == 0 {6} else {0};
    for col in (startcol..width).step(6) {
      let pos = if col == 0 {
        // At start of line predictor starts with first pixel of start of previous line
        (row-1)*width
      } else {
        // All other cases use the last pixel in the same two lines
        row*width+col-3
      };
      let (py,pcb,pcr) = (out[pos],out[pos+1],out[pos+2]);

      let y1 = (py  as i32) + try!(htable1.huff_decode(&mut pump));
      let y2 = (y1  as i32) + try!(htable1.huff_decode(&mut pump));
      let cb = (pcb as i32) + try!(htable2.huff_decode(&mut pump));
      let cr = (pcr as i32) + try!(htable3.huff_decode(&mut pump));
      set_yuv_422(out, row, col, width, y1, y2, cb, cr);
    }
  }

  Ok(())
}

pub fn decode_hasselblad(ljpeg: &LjpegDecompressor, out: &mut [u16], width: usize) -> Result<(),String> {
  // Pixels are packed two at a time, not like LJPEG:
  // [p1_length_as_huffman][p2_length_as_huffman][p0_diff_with_length][p1_diff_with_length]|NEXT PIXELS
  let mut pump = BitPumpMSB32::new(ljpeg.buffer);
  let ref htable = ljpeg.dhts[ljpeg.sof.components[0].dc_tbl_num];

  for line in out.chunks_mut(width) {
    let mut p1: i32 = 0x8000;
    let mut p2: i32 = 0x8000;
    for o in line.chunks_mut(2) {
      let len1 = try!(htable.huff_len(&mut pump));
      let len2 = try!(htable.huff_len(&mut pump));
      p1 += htable.huff_diff(&mut pump, len1);
      p2 += htable.huff_diff(&mut pump, len2);
      o[0] = p1 as u16;
      o[1] = p2 as u16;
    }
  }

  Ok(())
}
<commit_msg>ljpeg: make the 422 and 420 size checks strict<commit_after>use decoders::ljpeg::*;
use decoders::basics::*;
use itertools::Itertools;

pub fn decode_ljpeg_2components(ljpeg: &LjpegDecompressor, out: &mut [u16], x: usize, stripwidth:usize, width: usize, height: usize) -> Result<(),String> {
  if ljpeg.sof.width*2 < width || ljpeg.sof.height < height {
    return Err(format!("ljpeg: trying to decode {}x{} into {}x{}",
                       ljpeg.sof.width*2, ljpeg.sof.height,
                       width, height).to_string())
  }
  let ref htable1 = ljpeg.dhts[ljpeg.sof.components[0].dc_tbl_num];
  let ref htable2 = ljpeg.dhts[ljpeg.sof.components[1].dc_tbl_num];
  let mut pump = BitPumpJPEG::new(ljpeg.buffer);

  let base_prediction = 1 << (ljpeg.sof.precision - ljpeg.point_transform -1);
  out[x]   = (base_prediction + try!(htable1.huff_decode(&mut pump))) as u16;
  out[x+1] = (base_prediction + try!(htable2.huff_decode(&mut pump))) as u16;
  let skip_x = ljpeg.sof.width - width/2;

  for row in 0..height {
    let startcol = if row == 0 {x+2} else {x};
    for col in (startcol..(width+x)).step(2) {
      let (p1,p2) = if col == x {
        // At start of line predictor starts with start of previous line
        (out[(row-1)*stripwidth+x],out[(row-1)*stripwidth+1+x])
      } else {
        // All other cases use the two previous pixels in the same line
        (out[row*stripwidth+col-2], out[row*stripwidth+col-1])
      };

      let diff1 = try!(htable1.huff_decode(&mut pump));
      let diff2 = try!(htable2.huff_decode(&mut pump));
      out[row*stripwidth+col] = ((p1 as i32) + diff1) as u16;
      out[row*stripwidth+col+1] = ((p2 as i32) + diff2) as u16;
    }
    // Skip extra encoded differences if the ljpeg frame is wider than the output
    for _ in 0..skip_x {
      try!(htable1.huff_decode(&mut pump));
      try!(htable2.huff_decode(&mut pump));
    }
  }

  Ok(())
}

pub fn decode_ljpeg_4components(ljpeg: &LjpegDecompressor, out: &mut [u16], width: usize, height: usize) -> Result<(),String> {
  if ljpeg.sof.width*4 < width || ljpeg.sof.height < height {
    return Err(format!("ljpeg: trying to decode {}x{} into {}x{}",
                       ljpeg.sof.width*4, ljpeg.sof.height,
                       width, height).to_string())
  }
  let ref htable1 = ljpeg.dhts[ljpeg.sof.components[0].dc_tbl_num];
  let ref htable2 = ljpeg.dhts[ljpeg.sof.components[1].dc_tbl_num];
  let ref htable3 = ljpeg.dhts[ljpeg.sof.components[2].dc_tbl_num];
  let ref htable4 = ljpeg.dhts[ljpeg.sof.components[3].dc_tbl_num];
  let mut pump = BitPumpJPEG::new(ljpeg.buffer);

  let base_prediction = 1 << (ljpeg.sof.precision - ljpeg.point_transform -1);
  out[0] = (base_prediction + try!(htable1.huff_decode(&mut pump))) as u16;
  out[1] = (base_prediction + try!(htable2.huff_decode(&mut pump))) as u16;
  out[2] = (base_prediction + try!(htable3.huff_decode(&mut pump))) as u16;
  out[3] = (base_prediction + try!(htable4.huff_decode(&mut pump))) as u16;
  let skip_x = ljpeg.sof.width - width/4;

  for row in 0..height {
    let startcol = if row == 0 {4} else {0};
    for col in (startcol..width).step(4) {
      let pos = if col == 0 {
        // At start of line predictor starts with start of previous line
        (row-1)*width
      } else {
        // All other cases use the two previous pixels in the same line
        row*width+col-4
      };

      let (p1,p2,p3,p4) = (out[pos],out[pos+1],out[pos+2],out[pos+3]);

      let diff1 = try!(htable1.huff_decode(&mut pump));
      let diff2 = try!(htable2.huff_decode(&mut pump));
      let diff3 = try!(htable3.huff_decode(&mut pump));
      let diff4 = try!(htable4.huff_decode(&mut pump));
      out[row*width+col] = ((p1 as i32) + diff1) as u16;
      out[row*width+col+1] = ((p2 as i32) + diff2) as u16;
      out[row*width+col+2] = ((p3 as i32) + diff3) as u16;
      out[row*width+col+3] = ((p4 as i32) + diff4) as u16;
    }
    // Skip extra encoded differences if the ljpeg frame is wider than the output
    for _ in 0..skip_x {
      try!(htable1.huff_decode(&mut pump));
      try!(htable2.huff_decode(&mut pump));
      try!(htable3.huff_decode(&mut pump));
      try!(htable4.huff_decode(&mut pump));
    }
  }

  Ok(())
}

fn set_yuv_420(out: &mut [u16], row: usize, col: usize, width: usize, y1: i32, y2: i32, y3: i32, y4: i32, cb: i32, cr: i32) {
  let pix1 = row*width+col;
  let pix2 = pix1+3;
  let pix3 = (row+1)*width+col;
  let pix4 = pix3+3;

  out[pix1+0] = y1 as u16;
  out[pix1+1] = cb as u16;
  out[pix1+2] = cr as u16;
  out[pix2+0] = y2 as u16;
  out[pix2+1] = cb as u16;
  out[pix2+2] = cr as u16;
  out[pix3+0] = y3 as u16;
  out[pix3+1] = cb as u16;
  out[pix3+2] = cr as u16;
  out[pix4+0] = y4 as u16;
  out[pix4+1] = cb as u16;
  out[pix4+2] = cr as u16;
}

pub fn decode_ljpeg_420(ljpeg: &LjpegDecompressor, out: &mut [u16], width: usize, height: usize) -> Result<(),String> {
  if ljpeg.sof.width*3 != width || ljpeg.sof.height != height {
    return Err(format!("ljpeg: trying to decode {}x{} into {}x{}",
                       ljpeg.sof.width*3, ljpeg.sof.height,
                       width, height).to_string())
  }

  let ref htable1 = ljpeg.dhts[ljpeg.sof.components[0].dc_tbl_num];
  let ref htable2 = ljpeg.dhts[ljpeg.sof.components[1].dc_tbl_num];
  let ref htable3 = ljpeg.dhts[ljpeg.sof.components[2].dc_tbl_num];
  let mut pump = BitPumpJPEG::new(ljpeg.buffer);

  let base_prediction = 1 << (ljpeg.sof.precision - ljpeg.point_transform -1);
  let y1 = base_prediction + try!(htable1.huff_decode(&mut pump));
  let y2 = y1 + try!(htable1.huff_decode(&mut pump));
  let y3 = y2 + try!(htable1.huff_decode(&mut pump));
  let y4 = y3 + try!(htable1.huff_decode(&mut pump));
  let cb = base_prediction + try!(htable2.huff_decode(&mut pump));
  let cr = base_prediction + try!(htable3.huff_decode(&mut pump));
  set_yuv_420(out, 0, 0, width, y1, y2, y3, y4, cb, cr);

  for row in (0..height).step(2) {
    let startcol = if row == 0 {6} else {0};
    for col in (startcol..width).step(6) {
      let pos = if col == 0 {
        // At start of line predictor starts with first pixel of start of previous line
        (row-2)*width
      } else {
        // All other cases use the last pixel in the same two lines
        (row+1)*width+col-3
      };
      let (py,pcb,pcr) = (out[pos],out[pos+1],out[pos+2]);

      let y1 = (py  as i32) + try!(htable1.huff_decode(&mut pump));
      let y2 = (y1  as i32) + try!(htable1.huff_decode(&mut pump));
      let y3 = (y2  as i32) + try!(htable1.huff_decode(&mut pump));
      let y4 = (y3  as i32) + try!(htable1.huff_decode(&mut pump));
      let cb = (pcb as i32) + try!(htable2.huff_decode(&mut pump));
      let cr = (pcr as i32) + try!(htable3.huff_decode(&mut pump));
      set_yuv_420(out, row, col, width, y1, y2, y3, y4, cb, cr);
    }
  }

  Ok(())
}

fn set_yuv_422(out: &mut [u16], row: usize, col: usize, width: usize, y1: i32, y2: i32, cb: i32, cr: i32) {
  let pix1 = row*width+col;
  let pix2 = pix1+3;

  out[pix1+0] = y1 as u16;
  out[pix1+1] = cb as u16;
  out[pix1+2] = cr as u16;
  out[pix2+0] = y2 as u16;
  out[pix2+1] = cb as u16;
  out[pix2+2] = cr as u16;
}

pub fn decode_ljpeg_422(ljpeg: &LjpegDecompressor, out: &mut [u16], width: usize, height: usize) -> Result<(),String> {
  if ljpeg.sof.width*3 != width || ljpeg.sof.height != height {
    return Err(format!("ljpeg: trying to decode {}x{} into {}x{}",
                       ljpeg.sof.width*3, ljpeg.sof.height,
                       width, height).to_string())
  }
  let ref htable1 = ljpeg.dhts[ljpeg.sof.components[0].dc_tbl_num];
  let ref htable2 = ljpeg.dhts[ljpeg.sof.components[1].dc_tbl_num];
  let ref htable3 = ljpeg.dhts[ljpeg.sof.components[2].dc_tbl_num];
  let mut pump = BitPumpJPEG::new(ljpeg.buffer);

  let base_prediction = 1 << (ljpeg.sof.precision - ljpeg.point_transform -1);
  let y1 = base_prediction + try!(htable1.huff_decode(&mut pump));
  let y2 = y1 + try!(htable1.huff_decode(&mut pump));
  let cb = base_prediction + try!(htable2.huff_decode(&mut pump));
  let cr = base_prediction + try!(htable3.huff_decode(&mut pump));
  set_yuv_422(out, 0, 0, width, y1, y2, cb, cr);

  for row in 0..height {
    let startcol = if row == 0 {6} else {0};
    for col in (startcol..width).step(6) {
      let pos = if col == 0 {
        // At start of line predictor starts with first pixel of start of previous line
        (row-1)*width
      } else {
        // All other cases use the last pixel in the same two lines
        row*width+col-3
      };
      let (py,pcb,pcr) = (out[pos],out[pos+1],out[pos+2]);

      let y1 = (py  as i32) + try!(htable1.huff_decode(&mut pump));
      let y2 = (y1  as i32) + try!(htable1.huff_decode(&mut pump));
      let cb = (pcb as i32) + try!(htable2.huff_decode(&mut pump));
      let cr = (pcr as i32) + try!(htable3.huff_decode(&mut pump));
      set_yuv_422(out, row, col, width, y1, y2, cb, cr);
    }
  }

  Ok(())
}

pub fn decode_hasselblad(ljpeg: &LjpegDecompressor, out: &mut [u16], width: usize) -> Result<(),String> {
  // Pixels are packed two at a time, not like LJPEG:
  // [p1_length_as_huffman][p2_length_as_huffman][p0_diff_with_length][p1_diff_with_length]|NEXT PIXELS
  let mut pump = BitPumpMSB32::new(ljpeg.buffer);
  let ref htable = ljpeg.dhts[ljpeg.sof.components[0].dc_tbl_num];

  for line in out.chunks_mut(width) {
    let mut p1: i32 = 0x8000;
    let mut p2: i32 = 0x8000;
    for o in line.chunks_mut(2) {
      let len1 = try!(htable.huff_len(&mut pump));
      let len2 = try!(htable.huff_len(&mut pump));
      p1 += htable.huff_diff(&mut pump, len1);
      p2 += htable.huff_diff(&mut pump, len2);
      o[0] = p1 as u16;
      o[1] = p2 as u16;
    }
  }

  Ok(())
}
<|endoftext|>


<commit_before>//! This module contains the underlying data generation and shrinking
//! mechanism. The main type is the `InfoPool`, which represents a pool of
//! random bytes that can be observed via the `InfoTap` object (obtained via
//! `InfoPool#tap`).
//!
//! Also manages the shrinking process (see [`minimize`](fn.minimize.html)).

use std::fmt;
use hex_slice::AsHex;
use rand::{random, Rng, XorShiftRng};
use std::cmp::min;

/// A pool of data that we can draw upon to generate other types of data.
#[derive(Clone, PartialEq)]
pub struct InfoPool {
    data: Vec<u8>,
}

/// A handle to an info Pool that we can draw newly generated bytes from.
pub struct InfoTap<'a> {
    data: &'a mut Vec<u8>,
    rng: XorShiftRng,
}

/// A handle to an info Pool that we can draw replayed bytes from, and zero after.
#[derive(Clone)]
pub struct InfoReplay<'a> {
    data: &'a [u8],
    off: usize,
}

impl fmt::Debug for InfoPool {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("InfoPool")
            .field("data", &format_args!("{:x}", self.data.as_hex()))
            .finish()
    }
}

/// The reasons why drawing data from a pool can fail.
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum DataError {
    /// Not enough data to generate a value
    PoolExhausted,
    /// One of our combinators said that we should not test this value.
    SkipItem,
}

impl InfoPool {
    /// Create an `InfoPool` with a given vector of bytes. (Mostly used for
    /// testing).
    pub fn of_vec(data: Vec<u8>) -> Self {
        InfoPool { data: data }
    }

    /// Create an `InfoPool` with a `size` length vector of random bytes
    /// using the generator `rng`. (Mostly used for testing).
    pub fn new() -> Self {
        Self { data: Vec::new() }
    }

    /// Allows access to the underlying buffer.
    pub fn buffer(&self) -> &[u8] {
        &*self.data
    }

    fn tap_with(&mut self, rng: XorShiftRng) -> InfoTap {
        InfoTap {
            data: &mut self.data,
            rng: rng,
        }
    }
    /// Creates a tap that allows drawing new information from this pool.
    pub fn tap(&mut self) -> InfoTap {
        self.tap_with(random::<XorShiftRng>())
    }

    /// Creates a tap that allows drawing new information from this pool.
    pub fn replay(&self) -> InfoReplay {
        InfoReplay {
            data: &*self.data,
            off: 0,
        }
    }
}

impl<'a> InfoTap<'a> {
    /// Consumes the next byte from this tap. Returns `Ok(x)` if successful,
    /// or `Err(DataError::PoolExhausted)` if we have reached the end.
    pub fn next_byte(&mut self) -> u8 {
        let next = self.rng.gen::<u8>();
        self.data.push(next);
        next
    }
}

impl<'a> Iterator for InfoTap<'a> {
    type Item = u8;
    fn next(&mut self) -> Option<u8> {
        Some(self.next_byte())
    }
}

impl<'a> InfoReplay<'a> {
    /// Consumes the next byte from this tap. Returns `Ok(x)` if successful,
    /// or `Err(DataError::PoolExhausted)` if we have reached the end.
    pub fn next_byte(&mut self) -> u8 {
        let res = self.data.get(self.off).cloned();
        self.off += 1;
        res.unwrap_or(0)
    }
}

impl<'a> Iterator for InfoReplay<'a> {
    type Item = u8;
    fn next(&mut self) -> Option<u8> {
        Some(self.next_byte())
    }
}

/// Iterates over a series of shrunk pools. If we imagine that our buffer has
/// a sz of (1 << (log2sz-1)) < sz ≤ (1 << log2sz), then where:
/// f(x) = 1<<(log2sz-x) we want to cut out
/// chunks of:
/// ```
/// 0..f(0),
/// 0..f(1), f(1)..2f(1),
/// 0..f(2), f(2)..2f(2), 2f(2)..3f(2), 3f(2)..4f(2),
/// ```
///
/// In other words, we remove the whole lot, then first half, second half,
/// first quarter, second quarter, etc.
#[derive(Debug)]
struct RemovalShrinker {
    seed: InfoPool,
    log2sz: usize,
    level: usize,
    chunk: usize,
}

impl RemovalShrinker {
    fn new(seed: InfoPool) -> Self {
        let max_pow = 0usize.count_zeros();
        let pow = max_pow - seed.data.len().leading_zeros();
        RemovalShrinker {
            seed,
            log2sz: pow as usize,
            // Ranges from 0..self.log2sz
            level: 0,
            // Ranges from 0..(1<<self.level)
            chunk: 0,
        }
    }
}

impl Iterator for RemovalShrinker {
    type Item = InfoPool;
    fn next(&mut self) -> Option<Self::Item> {
        loop {
            trace!("RemovalShrinker#next: {:?}", self);
            if self.level > self.log2sz {
                return None;
            }

            let granularity = self.log2sz - self.level;
            let width = 1 << granularity;
            let chunk = self.chunk;

            let start = chunk * width;
            let end = start + width;

            if start >= self.seed.data.len() {
                trace!(
                    "Out of slice ({},{}) >= {}",
                    start,
                    end,
                    self.seed.data.len()
                );
                self.chunk = 0;
                self.level += 1;
                continue;
            } else {
                self.chunk += 1;
            }
            let start = min(start, self.seed.data.len());
            let end = min(end, self.seed.data.len());

            let mut candidate = InfoPool::new();
            candidate.data.clear();
            candidate.data.extend(&self.seed.data[0..start]);
            candidate.data.extend(&self.seed.data[end..]);
            trace!("removed {},{} -> {:?}", start, end, candidate);

            return Some(candidate);
        }
    }
}

fn minimize_via_removal<F: Fn(InfoReplay) -> bool>(p: &InfoPool, pred: &F) -> Option<InfoPool> {
    let res = RemovalShrinker::new(p.clone())
        .filter(|c| {
            let test = pred(c.replay());
            trace!("test result: {:?} <= {:?}", test, c);
            test
        })
        .next();

    return res.map(|candidate| {
        trace!("Re-Shrinking: {:?}", candidate);
        return minimize(&candidate, pred).unwrap_or(candidate);
    });
}

fn minimize_via_scalar_shrink<F: Fn(InfoReplay) -> bool>(
    p: &InfoPool,
    candidate: &mut InfoPool,
    pred: &F,
) -> Option<InfoPool> {
    // Second shrink tactic: make values smaller
    trace!("minimizing by scalar shrink: {:?}", p);
    for i in 0..p.data.len() {
        candidate.clone_from(&p);

        let max_pow = 0u8.count_zeros();
        let pow = max_pow - p.data[i].leading_zeros();
        for bitoff in 0..pow {
            candidate.data[i] = p.data[i] - (p.data[i] >> bitoff);
            trace!(
                "shrunk item -(bitoff:{}) {} {}->{}: {:?}",
                bitoff,
                i,
                p.data[i],
                candidate.data[i],
                candidate
            );

            if candidate.buffer() == p.buffer() {
                trace!("No change");
                continue;
            }

            let test = pred(candidate.replay());
            trace!("test result {}", test);
            if test {
                if let Some(res) = minimize(&candidate, pred) {
                    trace!("Returning shrunk: {:?}", res);
                    return Some(res);
                } else {
                    trace!("Returning original: {:?}", candidate);
                    return Some(candidate.clone());
                }
            }
        }
    }

    None
}

/// Try to find the smallest pool `p` such that the predicate `pred` returns
/// true. Given that our [generators](../generators/index.html) tend to
/// generate smaller outputs from smaller inputs, by minimizing the source
/// pool we can find the smallest value that provokes a failure.
///
/// If we do not find any smaller pool that satisifies `pred`; we return
/// `None`.
///
/// Currently, we have two heuristics for shrinking: removing slices, and
/// reducing individual values.
///
/// Removing slices tries to remove as much of the pool as it can whilst still
/// having the predicate hold. At present, we do this by removing each half
/// and testing, then each quarter, eighths, and so on. Eventually, we plan to
/// track the regions of data that each generator draws from, and use that to
/// optimise the shrinking process.
///
/// Reducing individual values basically goes through each position in the
/// pool, and then tries reducing it to zero, then half, thn three quarters,
/// seven eighths, and so on.
pub fn minimize<F: Fn(InfoReplay) -> bool>(p: &InfoPool, pred: &F) -> Option<InfoPool> {
    if let Some(res) = minimize_via_removal(p, pred) {
        return Some(res);
    }

    let mut candidate = p.clone();
    if let Some(res) = minimize_via_scalar_shrink(p, &mut candidate, pred) {
        return Some(res);
    }

    trace!("Nothing smaller found than {:?}", p);
    None
}

#[cfg(test)]
mod tests {
    extern crate env_logger;
    use super::*;

    #[test]
    fn should_take_each_item_in_pool() {
        let p = InfoPool::of_vec(vec![0, 1, 2, 3]);
        let mut t = p.replay();
        assert_eq!(t.next_byte(), 0);
        assert_eq!(t.next_byte(), 1);
        assert_eq!(t.next_byte(), 2);
        assert_eq!(t.next_byte(), 3);
        assert_eq!(t.next_byte(), 0);
    }

    #[test]
    fn should_generate_random_data() {
        let trials = 1024usize;
        let mut vals = 0;
        let mut p = InfoPool::new();
        let mut t = p.tap_with(::rand::XorShiftRng::new_unseeded());
        let error = 4;
        for _ in 0..trials {
            vals += t.next_byte() as usize;
        }
        let mean = vals / trials;
        let expected = 128;
        assert!(
            (expected - error) < mean && (expected + error) > mean,
            "Expected {} trials to be ({}+/-{}); got {}",
            trials,
            expected,
            error,
            mean
        )
    }

    #[test]
    fn should_allow_restarting_read() {
        let mut p = InfoPool::new();
        let mut v0 = Vec::new();
        {
            let mut t = p.tap();
            for _ in 0..4 {
                v0.push(t.next_byte())
            }
        }

        let mut t = p.replay();
        let mut v1 = Vec::new();
        for _ in 0..4 {
            v1.push(t.next_byte())
        }

        assert_eq!(v0, v1)
    }

    #[test]
    fn should_allow_borrowing_buffer() {
        let p = InfoPool::of_vec(vec![1]);
        assert_eq!(p.buffer(), &[1]);
    }

    #[test]
    fn tap_can_act_as_iterator() {
        let buf = vec![4, 3, 2, 1];
        let p = InfoPool::of_vec(buf.clone());
        let _: &Iterator<Item = u8> = &p.replay();

        assert_eq!(p.replay().take(4).collect::<Vec<_>>(), buf)
    }

    #[test]
    fn replay_can_act_as_iterator() {
        let buf = vec![4, 3, 2, 1];
        let p = InfoPool::of_vec(buf.clone());
        let _: &Iterator<Item = u8> = &p.replay();

        assert_eq!(p.replay().take(4).collect::<Vec<_>>(), buf)
    }

    #[test]
    fn minimiser_should_minimise_to_empty() {
        let p = InfoPool::of_vec(vec![1]);
        let min = minimize(&p, &|_| true);

        assert_eq!(min.as_ref().map(|p| p.buffer()), Some([].as_ref()))
    }

    #[test]
    fn minimiser_should_minimise_to_minimum_given_size() {
        env_logger::init().unwrap_or(());
        let p = InfoPool::of_vec(vec![1; 4]);
        let min = minimize(&p, &|t| t.take(16).filter(|&v| v > 0).count() > 1)
            .expect("some smaller pool");

        assert_eq!(min.buffer(), &[1, 1])
    }

    #[test]
    fn minimiser_should_minimise_scalar_values() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| t.take(16).any(|v| v >= 3)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[3])
    }
    #[test]
    fn minimiser_should_minimise_scalar_values_to_empty() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| t.take(16).any(|_| true)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[] as &[u8])
    }

    #[test]
    fn minimiser_should_minimise_scalar_values_by_search() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| t.take(16).any(|v| v >= 13)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[13])
    }
    #[test]
    fn minimiser_should_minimise_scalar_values_accounting_for_overflow() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| t.take(16).any(|v| v >= 251)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[251])
    }
}
<commit_msg>Verify that RemovalShrinker results in unique outputs.<commit_after>//! This module contains the underlying data generation and shrinking
//! mechanism. The main type is the `InfoPool`, which represents a pool of
//! random bytes that can be observed via the `InfoTap` object (obtained via
//! `InfoPool#tap`).
//!
//! Also manages the shrinking process (see [`minimize`](fn.minimize.html)).

use std::fmt;
use hex_slice::AsHex;
use rand::{random, Rng, XorShiftRng};
use std::cmp::min;

/// A pool of data that we can draw upon to generate other types of data.
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct InfoPool {
    data: Vec<u8>,
}

/// A handle to an info Pool that we can draw newly generated bytes from.
pub struct InfoTap<'a> {
    data: &'a mut Vec<u8>,
    rng: XorShiftRng,
}

/// A handle to an info Pool that we can draw replayed bytes from, and zero after.
#[derive(Clone)]
pub struct InfoReplay<'a> {
    data: &'a [u8],
    off: usize,
}

impl fmt::Debug for InfoPool {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("InfoPool")
            .field("data", &format_args!("{:x}", self.data.as_hex()))
            .finish()
    }
}

/// The reasons why drawing data from a pool can fail.
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum DataError {
    /// Not enough data to generate a value
    PoolExhausted,
    /// One of our combinators said that we should not test this value.
    SkipItem,
}

impl InfoPool {
    /// Create an `InfoPool` with a given vector of bytes. (Mostly used for
    /// testing).
    pub fn of_vec(data: Vec<u8>) -> Self {
        InfoPool { data: data }
    }

    /// Create an `InfoPool` with a `size` length vector of random bytes
    /// using the generator `rng`. (Mostly used for testing).
    pub fn new() -> Self {
        Self { data: Vec::new() }
    }

    /// Allows access to the underlying buffer.
    pub fn buffer(&self) -> &[u8] {
        &*self.data
    }

    fn tap_with(&mut self, rng: XorShiftRng) -> InfoTap {
        InfoTap {
            data: &mut self.data,
            rng: rng,
        }
    }
    /// Creates a tap that allows drawing new information from this pool.
    pub fn tap(&mut self) -> InfoTap {
        self.tap_with(random::<XorShiftRng>())
    }

    /// Creates a tap that allows drawing new information from this pool.
    pub fn replay(&self) -> InfoReplay {
        InfoReplay {
            data: &*self.data,
            off: 0,
        }
    }
}

impl<'a> InfoTap<'a> {
    /// Consumes the next byte from this tap. Returns `Ok(x)` if successful,
    /// or `Err(DataError::PoolExhausted)` if we have reached the end.
    pub fn next_byte(&mut self) -> u8 {
        let next = self.rng.gen::<u8>();
        self.data.push(next);
        next
    }
}

impl<'a> Iterator for InfoTap<'a> {
    type Item = u8;
    fn next(&mut self) -> Option<u8> {
        Some(self.next_byte())
    }
}

impl<'a> InfoReplay<'a> {
    /// Consumes the next byte from this tap. Returns `Ok(x)` if successful,
    /// or `Err(DataError::PoolExhausted)` if we have reached the end.
    pub fn next_byte(&mut self) -> u8 {
        let res = self.data.get(self.off).cloned();
        self.off += 1;
        res.unwrap_or(0)
    }
}

impl<'a> Iterator for InfoReplay<'a> {
    type Item = u8;
    fn next(&mut self) -> Option<u8> {
        Some(self.next_byte())
    }
}

/// Iterates over a series of shrunk pools. If we imagine that our buffer has
/// a sz of (1 << (log2sz-1)) < sz ≤ (1 << log2sz), then where:
/// f(x) = 1<<(log2sz-x) we want to cut out
/// chunks of:
/// ```
/// 0..f(0),
/// 0..f(1), f(1)..2f(1),
/// 0..f(2), f(2)..2f(2), 2f(2)..3f(2), 3f(2)..4f(2),
/// ```
///
/// In other words, we remove the whole lot, then first half, second half,
/// first quarter, second quarter, etc.
#[derive(Debug)]
struct RemovalShrinker {
    seed: InfoPool,
    log2sz: usize,
    level: usize,
    chunk: usize,
}

impl RemovalShrinker {
    fn new(seed: InfoPool) -> Self {
        let max_idx = seed.data.len().saturating_sub(1);
        let max_pow = 0usize.count_zeros();
        let pow = max_pow - max_idx.leading_zeros();
        RemovalShrinker {
            seed,
            log2sz: pow as usize,
            // Ranges from 0..self.log2sz
            level: 0,
            // Ranges from 0..(1<<self.level)
            chunk: 0,
        }
    }
}

impl Iterator for RemovalShrinker {
    type Item = InfoPool;
    fn next(&mut self) -> Option<Self::Item> {
        loop {
            trace!("RemovalShrinker#next: {:?}", self);
            if self.level > self.log2sz {
                return None;
            }

            let granularity = self.log2sz - self.level;
            let width = 1 << granularity;
            let chunk = self.chunk;

            let start = chunk * width;
            let end = start + width;

            if start >= self.seed.data.len() {
                trace!(
                    "Out of slice ({},{}) >= {}",
                    start,
                    end,
                    self.seed.data.len()
                );
                self.chunk = 0;
                self.level += 1;
                continue;
            } else {
                self.chunk += 1;
            }
            let start = min(start, self.seed.data.len());
            let end = min(end, self.seed.data.len());

            let mut candidate = InfoPool::new();
            candidate.data.clear();
            candidate.data.extend(&self.seed.data[0..start]);
            candidate.data.extend(&self.seed.data[end..]);
            trace!("removed {},{} -> {:?}", start, end, candidate);

            return Some(candidate);
        }
    }
}

fn minimize_via_removal<F: Fn(InfoReplay) -> bool>(p: &InfoPool, pred: &F) -> Option<InfoPool> {
    let res = RemovalShrinker::new(p.clone())
        .filter(|c| {
            let test = pred(c.replay());
            trace!("test result: {:?} <= {:?}", test, c);
            test
        })
        .next();

    return res.map(|candidate| {
        trace!("Re-Shrinking: {:?}", candidate);
        return minimize(&candidate, pred).unwrap_or(candidate);
    });
}

fn minimize_via_scalar_shrink<F: Fn(InfoReplay) -> bool>(
    p: &InfoPool,
    candidate: &mut InfoPool,
    pred: &F,
) -> Option<InfoPool> {
    // Second shrink tactic: make values smaller
    trace!("minimizing by scalar shrink: {:?}", p);
    for i in 0..p.data.len() {
        candidate.clone_from(&p);

        let max_pow = 0u8.count_zeros();
        let pow = max_pow - p.data[i].leading_zeros();
        for bitoff in 0..pow {
            candidate.data[i] = p.data[i] - (p.data[i] >> bitoff);
            trace!(
                "shrunk item -(bitoff:{}) {} {}->{}: {:?}",
                bitoff,
                i,
                p.data[i],
                candidate.data[i],
                candidate
            );

            if candidate.buffer() == p.buffer() {
                trace!("No change");
                continue;
            }

            let test = pred(candidate.replay());
            trace!("test result {}", test);
            if test {
                if let Some(res) = minimize(&candidate, pred) {
                    trace!("Returning shrunk: {:?}", res);
                    return Some(res);
                } else {
                    trace!("Returning original: {:?}", candidate);
                    return Some(candidate.clone());
                }
            }
        }
    }

    None
}

/// Try to find the smallest pool `p` such that the predicate `pred` returns
/// true. Given that our [generators](../generators/index.html) tend to
/// generate smaller outputs from smaller inputs, by minimizing the source
/// pool we can find the smallest value that provokes a failure.
///
/// If we do not find any smaller pool that satisifies `pred`; we return
/// `None`.
///
/// Currently, we have two heuristics for shrinking: removing slices, and
/// reducing individual values.
///
/// Removing slices tries to remove as much of the pool as it can whilst still
/// having the predicate hold. At present, we do this by removing each half
/// and testing, then each quarter, eighths, and so on. Eventually, we plan to
/// track the regions of data that each generator draws from, and use that to
/// optimise the shrinking process.
///
/// Reducing individual values basically goes through each position in the
/// pool, and then tries reducing it to zero, then half, thn three quarters,
/// seven eighths, and so on.
pub fn minimize<F: Fn(InfoReplay) -> bool>(p: &InfoPool, pred: &F) -> Option<InfoPool> {
    if let Some(res) = minimize_via_removal(p, pred) {
        return Some(res);
    }

    let mut candidate = p.clone();
    if let Some(res) = minimize_via_scalar_shrink(p, &mut candidate, pred) {
        return Some(res);
    }

    trace!("Nothing smaller found than {:?}", p);
    None
}

#[cfg(test)]
mod tests {
    extern crate env_logger;
    use super::*;
    use std::collections::BTreeMap;

    #[test]
    fn should_take_each_item_in_pool() {
        let p = InfoPool::of_vec(vec![0, 1, 2, 3]);
        let mut t = p.replay();
        assert_eq!(t.next_byte(), 0);
        assert_eq!(t.next_byte(), 1);
        assert_eq!(t.next_byte(), 2);
        assert_eq!(t.next_byte(), 3);
        assert_eq!(t.next_byte(), 0);
    }

    #[test]
    fn should_generate_random_data() {
        let trials = 1024usize;
        let mut vals = 0;
        let mut p = InfoPool::new();
        let mut t = p.tap_with(::rand::XorShiftRng::new_unseeded());
        let error = 4;
        for _ in 0..trials {
            vals += t.next_byte() as usize;
        }
        let mean = vals / trials;
        let expected = 128;
        assert!(
            (expected - error) < mean && (expected + error) > mean,
            "Expected {} trials to be ({}+/-{}); got {}",
            trials,
            expected,
            error,
            mean
        )
    }

    #[test]
    fn should_allow_restarting_read() {
        let mut p = InfoPool::new();
        let mut v0 = Vec::new();
        {
            let mut t = p.tap();
            for _ in 0..4 {
                v0.push(t.next_byte())
            }
        }

        let mut t = p.replay();
        let mut v1 = Vec::new();
        for _ in 0..4 {
            v1.push(t.next_byte())
        }

        assert_eq!(v0, v1)
    }

    #[test]
    fn should_allow_borrowing_buffer() {
        let p = InfoPool::of_vec(vec![1]);
        assert_eq!(p.buffer(), &[1]);
    }

    #[test]
    fn tap_can_act_as_iterator() {
        let buf = vec![4, 3, 2, 1];
        let p = InfoPool::of_vec(buf.clone());
        let _: &Iterator<Item = u8> = &p.replay();

        assert_eq!(p.replay().take(4).collect::<Vec<_>>(), buf)
    }

    #[test]
    fn replay_can_act_as_iterator() {
        let buf = vec![4, 3, 2, 1];
        let p = InfoPool::of_vec(buf.clone());
        let _: &Iterator<Item = u8> = &p.replay();

        assert_eq!(p.replay().take(4).collect::<Vec<_>>(), buf)
    }

    #[test]
    fn minimiser_should_minimise_to_empty() {
        let p = InfoPool::of_vec(vec![1]);
        let min = minimize(&p, &|_| true);

        assert_eq!(min.as_ref().map(|p| p.buffer()), Some([].as_ref()))
    }

    #[test]
    fn minimiser_should_minimise_to_minimum_given_size() {
        env_logger::init().unwrap_or(());
        let p = InfoPool::of_vec(vec![1; 4]);
        let min = minimize(&p, &|t| t.take(16).filter(|&v| v > 0).count() > 1)
            .expect("some smaller pool");

        assert_eq!(min.buffer(), &[1, 1])
    }

    #[test]
    fn minimiser_should_minimise_scalar_values() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| t.take(16).any(|v| v >= 3)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[3])
    }
    #[test]
    fn minimiser_should_minimise_scalar_values_to_empty() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| t.take(16).any(|_| true)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[] as &[u8])
    }

    #[test]
    fn minimiser_should_minimise_scalar_values_by_search() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| t.take(16).any(|v| v >= 13)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[13])
    }
    #[test]
    fn minimiser_should_minimise_scalar_values_accounting_for_overflow() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| t.take(16).any(|v| v >= 251)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[251])
    }

    #[test]
    fn shrink_by_removal_should_produce_somewhat_outputs() {
        env_logger::init().unwrap_or(());
        let p = InfoPool::of_vec((0..256usize).map(|v| v as u8).collect::<Vec<_>>());
        let mut counts = BTreeMap::new();
        for val in RemovalShrinker::new(p) {
            debug!("{:?}", val);
            *counts.entry(val).or_insert(0) += 1;
        }

        assert!(
            counts.values().all(|&val| val == 1),
            "Expect all items to be unique; non-unique entries {:?}",
            counts
                .iter()
                .filter(|&(_, &v)| v != 1)
                .collect::<BTreeMap<_, _>>()
        )
    }
}
<|endoftext|>


<commit_before>#![feature(phase)]

#[phase(plugin, link)]
extern crate itertools;

use itertools::Itertools;

fn main() {
    println!("24 days of Rust - itertools (day 7)");
    let mut words = "hello supercalifragilisticexpialidocious programmer".words();
    words.apply(|word| println!("{} is {} characters long", word, word.len()));
}
<commit_msg>intersperse example<commit_after>#![feature(phase)]

#[phase(plugin, link)]
extern crate itertools;

use itertools::Itertools;

fn main() {
    println!("24 days of Rust - itertools (day 7)");
    let mut words = "hello supercalifragilisticexpialidocious programmer".words();
    words.apply(|word| println!("{} is {} characters long", word, word.len()));
    let it = range(1, 10u);
    println!("{}", it.intersperse(15u).collect::<Vec<_>>());
}
<|endoftext|>


<commit_before>use imgui::{Ui,ImStr,ImString};
use lichen::parse::{StreamParser,Env};

use std::fs;
use std::path::Path;

use app::AppState;

use std::io::{BufReader};
use std::fs::File;

pub struct FileState {
    /// MAX_PATH set to 260 chars
    cd: ImString,
    idx: i32,
    files: Vec<String>,
    selected: Option<String>,
    stream: Option<StreamParser<BufReader<File>>>,
}

impl Default for FileState {
    fn default() -> Self {
        let mut cd = ImString::from(im_str!("./"));
        cd.reserve(261); //arbitrary max path size

        FileState {
            cd: cd,
            idx: 1,
            files: vec![],
            selected: None,
            stream: None,
        }
    }
}

impl FileState {
    fn update(&mut self) -> Option<Env> {
        if self.idx > 1 { // chose a file?
            if let Some(file) = self.files.get(self.idx as usize) {
                self.cd.clear();
                self.cd.push_str(file);
                self.idx = 1;
            }
        }
        else if self.idx < 1 { // up a directory?
            self.to_parent();
        }

        let cd = self.cd.to_string();
        if Path::new(&cd).is_dir() {
            if let Ok(paths) = fs::read_dir(&cd) {
                self.files = paths
                    .filter(|p| p.is_ok())
                    .map(|p| {
                        p.unwrap().path().as_path().to_str().unwrap().to_owned()
                    })
                    .collect();

                self.files.insert(0,"./".to_owned());
                self.files.insert(0,"..".to_owned());
            }
            // NOTE: we should throw a warning here if directory cannot be traversed
        }
        else {
            if !cd.is_empty() {
                self.selected = Some(cd);
            }
        }

        if self.stream.is_some() { self.step_stream() }
        else { None }
    }

    /// Steps through the stream chunks, finally returns parsed lichen environment
    fn step_stream (&mut self) -> Option<Env> {
        let r = {
            if let Some(ref mut stream) = self.stream {
                stream.next()
            }
            else { None }
        };

        if r.is_none() { // close stream, we're probably done
            let mut env = Env::empty();
            if let Some(ref mut stream) = self.stream {
                stream.sink(&mut env);
            }

            self.stream = None;
            self.to_parent();

            return Some(env);
        }

        None
    }

    fn to_parent(&mut self) {
        self.selected = None;
        if let Some(path) = Path::new(&self.cd.to_string()).parent() {
            self.cd.clear();
            self.cd.push_str(path.to_str().expect("ERROR: Cannot parse path into string"));
            self.idx = 1;
        }
        // NOTE: this fails if we're at the top of a relative directory listing,
        // or the root of the drive/share
    }

    pub fn render (&mut self, ui: &Ui, state: &mut AppState) {
        if !state.open_file { return }
        state.env = self.update(); // NOTE: we should check if the environment is empty
        state.open_file = !state.env.is_some(); // close file if done

        ui.window(im_str!("Select source"))
            .always_auto_resize(true)
            .movable(true)
            .show_borders(true)
            .opened(&mut state.open_file)
            .build(||{
                ui.text(im_str!("Select a source file to be parsed and evaluated.\nSource files must be parsable through the lichen crate."));
                ui.separator();
                ui.input_text(im_str!("Current Directory"), &mut self.cd)
                    .enter_returns_true(true)
                    .build();
                ui.separator();

                // NOTE: to appease the borrow checker and have the proper argument for list_box
                // we must build the list in two steps
                let paths: Vec<ImString> = self.files.iter().map(|p| {
                    im_str!("{:}", p).to_owned()
                }).collect();

                let paths: Vec<&ImStr> = paths.iter().map(|p|p.as_ref()).collect();

                ui.list_box(im_str!("Files"),
                            &mut self.idx,
                            &paths[..],
                            paths.len() as i32);

                if let Some(ref file) = self.selected {
                    ui.text(im_str!("Select File {:}?",file));

                    if ui.small_button(im_str!("open")) {
                        if let Ok(f) = File::open(file) {
                            let f = BufReader::new(f);
                            let s = StreamParser::new(f,None);
                            self.stream = Some(s);
                        }
                    }
                }
            })
    }

}

<commit_msg>display failure<commit_after>use imgui::{Ui,ImStr,ImString};
use lichen::parse::{StreamParser,Env};

use std::fs;
use std::path::Path;

use app::AppState;

use std::io::{BufReader};
use std::fs::File;

pub struct FileState {
    /// MAX_PATH set to 260 chars
    cd: ImString,
    idx: i32,
    files: Vec<String>,
    selected: Option<String>,
    stream: Option<StreamParser<BufReader<File>>>,
    failed: String,
}

impl Default for FileState {
    fn default() -> Self {
        let mut cd = ImString::from(im_str!("./"));
        cd.reserve(261); //arbitrary max path size

        FileState {
            cd: cd,
            idx: 1,
            files: vec![],
            selected: None,
            stream: None,
            failed: "".to_owned(),
        }
    }
}

impl FileState {
    fn update(&mut self) -> Option<Env> {
        if self.idx > 1 { // chose a file?
            self.failed = "".to_owned();
            if let Some(file) = self.files.get(self.idx as usize) {
                self.cd.clear();
                self.cd.push_str(file);
                self.idx = 1;
            }
        }
        else if self.idx < 1 { // up a directory?
            self.failed = "".to_owned();
            self.to_parent();
        }

        let cd = self.cd.to_string();
        if Path::new(&cd).is_dir() {
            if let Ok(paths) = fs::read_dir(&cd) {
                self.files = paths
                    .filter(|p| p.is_ok())
                    .map(|p| {
                        p.unwrap().path().as_path().to_str().unwrap().to_owned()
                    })
                    .collect();

                self.files.insert(0,"./".to_owned());
                self.files.insert(0,"..".to_owned());
            }
            // NOTE: we should throw a warning here if directory cannot be traversed
        }
        else {
            if !cd.is_empty() {
                self.selected = Some(cd);
                self.failed = "".to_owned();
            }
        }

        if self.stream.is_some() { self.step_stream() }
        else { None }
    }

    /// Steps through the stream chunks, finally returns parsed lichen environment
    fn step_stream (&mut self) -> Option<Env> {
        let r = {
            if let Some(ref mut stream) = self.stream {
                stream.next()
            }
            else { None }
        };

        if r.is_none() { // close stream, we're probably done
            let mut env = Env::empty();
            if let Some(ref mut stream) = self.stream {
                if let Err(e) = stream.sink(&mut env) {
                    self.failed = e.to_owned();
                    return None;
                }
            }

            self.stream = None;
            self.to_parent();

            return Some(env);
        }

        None
    }

    fn to_parent(&mut self) {
        self.selected = None;
        if let Some(path) = Path::new(&self.cd.to_string()).parent() {
            self.cd.clear();
            self.cd.push_str(path.to_str().expect("ERROR: Cannot parse path into string"));
            self.idx = 1;
        }
        // NOTE: this fails if we're at the top of a relative directory listing,
        // or the root of the drive/share
    }

    pub fn render (&mut self, ui: &Ui, state: &mut AppState) {
        if !state.open_file { return }
        state.env = self.update(); // NOTE: we should check if the environment is empty
        state.open_file = !state.env.is_some(); // close file if done

        ui.window(im_str!("Select source"))
            .always_auto_resize(true)
            .movable(true)
            .show_borders(true)
            .opened(&mut state.open_file)
            .build(||{
                ui.text(im_str!("Select a source file to be parsed and evaluated.\nSource files must be parsable through the lichen crate."));
                ui.separator();
                ui.input_text(im_str!("Current Directory"), &mut self.cd)
                    .enter_returns_true(true)
                    .build();
                ui.separator();

                // NOTE: to appease the borrow checker and have the proper argument for list_box
                // we must build the list in two steps
                let paths: Vec<ImString> = self.files.iter().map(|p| {
                    im_str!("{:}", p).to_owned()
                }).collect();

                let paths: Vec<&ImStr> = paths.iter().map(|p|p.as_ref()).collect();

                ui.list_box(im_str!("Files"),
                            &mut self.idx,
                            &paths[..],
                            paths.len() as i32);

                if let Some(ref file) = self.selected {
                    ui.text(im_str!("Select File {:}?",file));

                    if ui.small_button(im_str!("open")) {
                        if let Ok(f) = File::open(file) {
                            let f = BufReader::new(f);
                            let s = StreamParser::new(f,None);
                            self.stream = Some(s);
                        }
                    }
                }

                let mut failure = "".to_owned();
                if !self.failed.is_empty() {
                    failure.push_str("Failed to parse file ");
                    failure.push_str(&self.cd.to_str());
                    failure.push_str("\nParse Error at: ");
                    failure.push_str(&self.failed);
                }

                ui.text(im_str!("{:}", failure));
            })
    }

}

<|endoftext|>