<commit_before>#[allow(unused_variables)]
extern crate fnv;
extern crate rand;
extern crate time;
extern crate timely;
extern crate differential_dataflow;

use std::hash::{Hash, Hasher};
use std::fmt::Debug;
use std::io::{BufReader, BufRead};
use std::fs::File;

use timely::Data;
use timely::dataflow::*;
use timely::dataflow::scopes::Child;
use timely::dataflow::operators::*;
use timely::dataflow::operators::feedback::{Observer, Helper};
use timely::dataflow::channels::pushers::Counter;
use timely::progress::nested::product::Product;
use timely::progress::nested::Summary::Local;
use timely::progress::timestamp::RootTimestamp;

use differential_dataflow::operators::*;
use differential_dataflow::collection::LeastUpperBound;

use fnv::FnvHasher;

fn hash<T: Hash>(x: &T) -> u64 {
    let mut h: FnvHasher = Default::default();
    x.hash(&mut h);
    h.finish()
}

/// A collection defined by multiple mutually recursive rules.
pub struct Variable<G: Scope, D: Ord+Default+Clone+Data+Debug+Hash>
where G::Timestamp: LeastUpperBound {
    feedback: Option<Helper<Product<G::Timestamp, u64>,(D, i32),Counter<Product<G::Timestamp, u64>,
                                         (D, i32),Observer<Product<G::Timestamp, u64>,(D, i32)>>>>,
    current:  Stream<Child<G, u64>, (D,i32)>,
}

impl<G: Scope, D: Ord+Default+Clone+Data+Debug+Hash> Variable<G, D> where G::Timestamp: LeastUpperBound {
    /// Creates a new `Variable` and a `Stream` representing its output, from a supplied `source` stream.
    pub fn from(source: &Stream<Child<G, u64>, (D,i32)>) -> (Variable<G, D>, Stream<Child<G,u64>, (D, i32)>) {
        let (feedback, cycle) = source.scope().loop_variable(Product::new(G::Timestamp::max(), u64::max_value()), Local(1));
        let mut result = Variable { feedback: Some(feedback), current: cycle.clone() };
        let stream = cycle.clone();
        result.add(source);
        (result, stream)
    }
    /// Adds a new source of data to the `Variable`.
    pub fn add(&mut self, source: &Stream<Child<G, u64>, (D,i32)>) {
        self.current = self.current.concat(source);
    }
}

impl<G: Scope, D: Ord+Default+Debug+Hash+Clone+Data> Drop for Variable<G, D> where G::Timestamp: LeastUpperBound {
    fn drop(&mut self) {
        if let Some(feedback) = self.feedback.take() {
            self.current.group_by(|x| (x, ()), hash, hash, |x,_| x.clone(), |x,_,t| t.push((x.clone(),1)))
                        .connect_loop(feedback);
        }
    }
}

macro_rules! rule {
    ($name1: ident ($($var1:ident),*) := $name2: ident ($($var2:ident),*) $name3: ident ($($var3:ident),*) : ($($var4:ident),*) = ($($var5:ident),*)) => {{
        let result =
            $name2.0.join_by(
                &$name3.0,
                |($( $var2, )*)| (($( $var4, )*), ( $($var2, )*)),
                |($( $var3, )*)| (($( $var5, )*), ( $($var3, )*)),
                hash,
                |_, &($( $var2, )*), &($( $var3, )*)| (($( $var2, )*), ($( $var3, )*)));
        $name1.1.add(&result.map(|((($( $var2, )*), ($( $var3, )*)), __w)| (($( $var1, )*), __w)));

        let temp = result.semijoin_by(&$name1.2, |(($( $var2, )*), ($( $var3, )*))| (($( $var1, )*), (($( $var2, )*), ($( $var3, )*))), hash, |_, &(($( $var2, )*), ($( $var3, )*))| (($( $var2, )*), ($( $var3, )*)));
        $name2.3.add(&temp.map(|(( ($( $var2, )*) ,_),__w)| (($( $var2, )*),__w)));
        $name3.3.add(&temp.map(|(( _, ($( $var3, )*)),__w)| (($( $var3, )*),__w)));

        temp
    }};
}

macro_rules! variable {
    ($name0: ident : $name1: ident, $name2: ident) => {{
        let temp1 = Variable::from(&$name0.enter(&$name1));
        let temp2 = Variable::from(&$name0.enter(&$name2));
        (temp1.1, temp1.0, temp2.1, temp2.0)
    }}
}

fn main() {

    timely::execute_from_args(std::env::args(), |root| {

        let start = time::precise_time_s();
        let (mut p, mut q, mut u, mut p_query, mut q_query, probe) = root.scoped::<u64, _, _>(move |outer| {

            // inputs for p, q, and u base facts.
            let (p_input, p) = outer.new_input();
            let (q_input, q) = outer.new_input();
            let (u_input, u) = outer.new_input();

            // inputs through which to demand explanations.
            let (p_query_input, p_query) = outer.new_input();
            let (q_query_input, q_query) = outer.new_input();

            // determine which rules fire with what variable settings.
            let (p_base, q_base, ir1, ir2, ir3) = outer.scoped::<u64, _, _>(|inner| {

                let (_unused, u) = Variable::from(&inner.enter(&u));

                let mut p = variable!(inner : p, p_query);
                let mut q = variable!(inner : q, q_query);

                let ir1 = rule!(p(x,z)   := p(x,_y1) p(_y2,z)   : (_y1) = (_y2));
                let ir2 = rule!(q(x,r,z) := p(x,_y1) q(_y2,r,z) : (_y1) = (_y2));

                // P(x,z) := P(y,w), Q(x,r,y), U(w,r,z)
                let ir3 = p.0.join_by_u(&q.0, |(y,w)| (y,w), |(x,r,y)| (y,(x,r)), |&y, &w, &(x,r)| (r,w,x,y))
                             .join_by(&u, |(r,w,x,y)| ((r,w), (y,x)), |(w,r,z)| ((r,w),z), hash, |&(r,w), &(y,x), &z| (r,w,x,y,z));
                p.1.add(&ir3.map(|((_,_,x,_,z),w)| ((x,z),w)));
                let ir3_need = ir3.semijoin_by(&p.2, |(r,w,x,y,z)| ((x,z), (r,w,y)), hash, |&(x,z),&(r,w,y)| (r,w,x,y,z));
                p.3.add(&ir3_need.map(|((_,w,_,y,_),w2)| ((y,w),w2)));
                q.3.add(&ir3_need.map(|((r,_,x,y,_),w2)| ((x,r,y),w2)));

                // extract the results and return
                (p.2.leave(), q.2.leave(), ir1.leave(), ir2.leave(), ir3.leave())
            });

            let (probe, _) = p_base.consolidate().probe();

            p_base.consolidate().inspect(|&(x,_w)| println!("Required P{:?}", x));
            q_base.consolidate().inspect(|&(x,_w)| println!("Required Q{:?}", x));

            ir1.consolidate().inspect(|&(x,_w)| println!("Required IR1 {:?}", x));
            ir2.consolidate().inspect(|&(x,_w)| println!("Required IR2 {:?}", x));
            ir3.consolidate().inspect(|&(x,_w)| println!("Required IR3 {:?}", x));

            (p_input, q_input, u_input, p_query_input, q_query_input, probe)
        });

        if root.index() == 0 {
            let p_file = BufReader::new(File::open("/Users/mcsherry/Desktop/p.txt").unwrap());
            for readline in p_file.lines() {
                let line = readline.ok().expect("read error");
                let elts: Vec<&str> = line[..].split(",").collect();
                let src: u32 = elts[0].parse().ok().expect("malformed src");
                let dst: u32 = elts[1].parse().ok().expect("malformed dst");
                p.send(((src, dst), 1));
            }

            let q_file = BufReader::new(File::open("/Users/mcsherry/Desktop/q.txt").unwrap());
            for readline in q_file.lines() {
                let line = readline.ok().expect("read error");
                let elts: Vec<&str> = line[..].split(",").collect();
                let src: u32 = elts[0].parse().ok().expect("malformed src");
                let dst: u32 = elts[1].parse().ok().expect("malformed dst");
                let aeo: u32 = elts[2].parse().ok().expect("malformed dst");
                q.send(((src, dst, aeo), 1));
            }

            let u_file = BufReader::new(File::open("/Users/mcsherry/Desktop/u.txt").unwrap());
            for readline in u_file.lines() {
                let line = readline.ok().expect("read error");
                let elts: Vec<&str> = line[..].split(",").collect();
                let src: u32 = elts[0].parse().ok().expect("malformed src");
                let dst: u32 = elts[1].parse().ok().expect("malformed dst");
                let aeo: u32 = elts[2].parse().ok().expect("malformed dst");
                u.send(((src, dst, aeo), 1));
            }
        }

        println!("loading:\t{}", time::precise_time_s() - start);

        p.close();
        q.close();
        u.close();

        p_query.advance_to(1);
        q_query.advance_to(1);

        while probe.lt(&RootTimestamp::new(1)) {
            root.step();
        }

        println!("derivation:\t{}", time::precise_time_s() - start);
        let timer = time::precise_time_s();

        p_query.send(((36465u32,10135u32), 1));
        p_query.advance_to(2);
        q_query.advance_to(2);

        while probe.lt(&RootTimestamp::new(2)) {
            root.step();
        }

        println!("query:\t{}", time::precise_time_s() - timer);

        p_query.close();
        q_query.close();

        while root.step() { }    // wind down the computation

    });
}
<commit_msg>updated to reflect feedback name change; now with macros<commit_after>#[allow(unused_variables)]
extern crate fnv;
extern crate rand;
extern crate time;
extern crate timely;
extern crate differential_dataflow;

use std::io::{BufReader, BufRead};
use std::fs::File;

use timely::dataflow::*;
use timely::dataflow::scopes::Child;
use timely::dataflow::operators::*;
use timely::dataflow::operators::feedback::Handle;
use timely::progress::timestamp::RootTimestamp;

use differential_dataflow::Data;
use differential_dataflow::operators::*;
use differential_dataflow::collection::LeastUpperBound;

/// A collection defined by multiple mutually recursive rules.
pub struct Variable<G: Scope, D: Default+Data>
where G::Timestamp: LeastUpperBound {
    feedback: Option<Handle<G::Timestamp, u64,(D, i32)>>,
    current:  Stream<Child<G, u64>, (D,i32)>,
}

impl<G: Scope, D: Default+Data> Variable<G, D> where G::Timestamp: LeastUpperBound {
    /// Creates a new `Variable` and a `Stream` representing its output, from a supplied `source` stream.
    pub fn from(source: &Stream<Child<G, u64>, (D,i32)>) -> (Variable<G, D>, Stream<Child<G,u64>, (D, i32)>) {
        let (feedback, cycle) = source.scope().loop_variable(u64::max_value(), 1);
        let mut result = Variable { feedback: Some(feedback), current: cycle.clone() };
        let stream = cycle.clone();
        result.add(source);
        (result, stream)
    }
    /// Adds a new source of data to the `Variable`.
    pub fn add(&mut self, source: &Stream<Child<G, u64>, (D,i32)>) {
        self.current = self.current.concat(source);
    }
}

impl<G: Scope, D: Default+Data> Drop for Variable<G, D> where G::Timestamp: LeastUpperBound {
    fn drop(&mut self) {
        if let Some(feedback) = self.feedback.take() {
            self.current.group_by(|x| (x, ()), |x| x.hashed(), |x| x.hashed(), |x,_| x.clone(), |_,_,t| t.push(((),1)))
                        .connect_loop(feedback);
        }
    }
}

macro_rules! rule {
    ($name1: ident ($($var1:ident),*) := $name2: ident ($($var2:ident),*) $name3: ident ($($var3:ident),*) : ($($var4:ident),*) = ($($var5:ident),*)) => {{
        let result =
            $name2.0.join_by(
                &$name3.0,
                |($( $var2, )*)| (($( $var4, )*), ( $($var2, )*)),
                |($( $var3, )*)| (($( $var5, )*), ( $($var3, )*)),
                |x| x.hashed(),
                |_, &($( $var2, )*), &($( $var3, )*)| (($( $var2, )*), ($( $var3, )*)));
        $name1.1.add(&result.map(|((($( $var2, )*), ($( $var3, )*)), __w)| (($( $var1, )*), __w)));

        let temp = result.semijoin_by(
            &$name1.2,
            |(($( $var2, )*), ($( $var3, )*))| (($( $var1, )*), (($( $var2, )*), ($( $var3, )*))),
            |x| x.hashed(),
            |_, &(($( $var2, )*), ($( $var3, )*))| (($( $var2, )*), ($( $var3, )*)));
        $name2.3.add(&temp.map(|(( ($( $var2, )*) ,_),__w)| (($( $var2, )*),__w)));
        $name3.3.add(&temp.map(|(( _, ($( $var3, )*)),__w)| (($( $var3, )*),__w)));

        temp
    }};
}

macro_rules! rule_u {
    ($name1: ident ($($var1:ident),*) := $name2: ident ($($var2:ident),*) $name3: ident ($($var3:ident),*) : $var4:ident = $var5:ident) => {{
        let result =
            $name2.0.join_by_u(
                &$name3.0,
                |($( $var2, )*)| ($var4, ( $($var2, )*)),
                |($( $var3, )*)| ($var5, ( $($var3, )*)),
                |_, &($( $var2, )*), &($( $var3, )*)| (($( $var2, )*), ($( $var3, )*)));
        $name1.1.add(&result.map(|((($( $var2, )*), ($( $var3, )*)), __w)| (($( $var1, )*), __w)));

        let temp = result.semijoin_by(
            &$name1.2,
            |(($( $var2, )*), ($( $var3, )*))| (($( $var1, )*), (($( $var2, )*), ($( $var3, )*))),
            |x| x.hashed(),
            |_, &(($( $var2, )*), ($( $var3, )*))| (($( $var2, )*), ($( $var3, )*)));
        $name2.3.add(&temp.map(|(( ($( $var2, )*) ,_),__w)| (($( $var2, )*),__w)));
        $name3.3.add(&temp.map(|(( _, ($( $var3, )*)),__w)| (($( $var3, )*),__w)));

        temp
    }};
}

macro_rules! variable {
    ($name0: ident : $name1: ident, $name2: ident) => {{
        let temp1 = Variable::from(&$name0.enter(&$name1));
        let temp2 = Variable::from(&$name0.enter(&$name2));
        (temp1.1, temp1.0, temp2.1, temp2.0)
    }}
}

fn main() {

    timely::execute_from_args(std::env::args(), |root| {

        let start = time::precise_time_s();
        let (mut p, mut q, mut u, mut p_query, mut q_query, probe) = root.scoped::<u64, _, _>(move |outer| {

            // inputs for p, q, and u base facts.
            let (p_input, p) = outer.new_input();
            let (q_input, q) = outer.new_input();
            let (u_input, u) = outer.new_input();

            // inputs through which to demand explanations.
            let (p_query_input, p_query) = outer.new_input();
            let (q_query_input, q_query) = outer.new_input();

            // determine which rules fire with what variable settings.
            let (p_base, q_base, ir1, ir2, ir3) = outer.scoped::<u64, _, _>(|inner| {

                let (_unused, u) = Variable::from(&inner.enter(&u));

                let mut p = variable!(inner : p, p_query);
                let mut q = variable!(inner : q, q_query);

                let ir1 = rule_u!(p(x,z)   := p(x,_y1) p(_y2,z)   : _y1 = _y2);
                let ir2 = rule_u!(q(x,r,z) := p(x,_y1) q(_y2,r,z) : _y1 = _y2);

                // P(x,z) := P(y,w), Q(x,r,y), U(w,r,z)
                let ir3 = p.0.join_by_u(&q.0, |(y,w)| (y,w), |(x,r,y)| (y,(x,r)), |&y, &w, &(x,r)| (r,w,x,y))
                             .join_by(&u, |(r,w,x,y)| ((r,w), (y,x)), |(w,r,z)| ((r,w),z), |x| x.hashed(), |&(r,w), &(y,x), &z| (r,w,x,y,z));
                p.1.add(&ir3.map(|((_,_,x,_,z),w)| ((x,z),w)));
                let ir3_need = ir3.semijoin_by(&p.2, |(r,w,x,y,z)| ((x,z), (r,w,y)), |x| x.hashed(), |&(x,z),&(r,w,y)| (r,w,x,y,z));
                p.3.add(&ir3_need.map(|((_,w,_,y,_),w2)| ((y,w),w2)));
                q.3.add(&ir3_need.map(|((r,_,x,y,_),w2)| ((x,r,y),w2)));

                // extract the results and return
                (p.2.leave(), q.2.leave(), ir1.leave(), ir2.leave(), ir3_need.leave())
            });

            let (probe, _) = p_base.consolidate().probe();

            // p_base.consolidate().inspect(|&(x,_w)| println!("Required P{:?}", x));
            // q_base.consolidate().inspect(|&(x,_w)| println!("Required Q{:?}", x));

            ir1.consolidate().inspect(|&(x,_w)| println!("Required IR1 {:?}", x));
            ir2.consolidate().inspect(|&(x,_w)| println!("Required IR2 {:?}", x));
            ir3.consolidate().inspect(|&(x,_w)| println!("Required IR3 {:?}", x));

            (p_input, q_input, u_input, p_query_input, q_query_input, probe)
        });

        if root.index() == 0 {
            let p_file = BufReader::new(File::open("/Users/mcsherry/Desktop/p.txt").unwrap());
            for readline in p_file.lines() {
                let line = readline.ok().expect("read error");
                let elts: Vec<&str> = line[..].split(",").collect();
                let src: u32 = elts[0].parse().ok().expect("malformed src");
                let dst: u32 = elts[1].parse().ok().expect("malformed dst");
                if src != dst {
                    p.send(((src, dst), 1));
                }
            }

            let q_file = BufReader::new(File::open("/Users/mcsherry/Desktop/q.txt").unwrap());
            for readline in q_file.lines() {
                let line = readline.ok().expect("read error");
                let elts: Vec<&str> = line[..].split(",").collect();
                let src: u32 = elts[0].parse().ok().expect("malformed src");
                let dst: u32 = elts[1].parse().ok().expect("malformed dst");
                let aeo: u32 = elts[2].parse().ok().expect("malformed dst");
                q.send(((src, dst, aeo), 1));
            }

            let u_file = BufReader::new(File::open("/Users/mcsherry/Desktop/u.txt").unwrap());
            for readline in u_file.lines() {
                let line = readline.ok().expect("read error");
                let elts: Vec<&str> = line[..].split(",").collect();
                let src: u32 = elts[0].parse().ok().expect("malformed src");
                let dst: u32 = elts[1].parse().ok().expect("malformed dst");
                let aeo: u32 = elts[2].parse().ok().expect("malformed dst");
                u.send(((src, dst, aeo), 1));
            }
        }

        println!("loading:\t{}", time::precise_time_s() - start);

        p.close();
        q.close();
        u.close();

        p_query.advance_to(1);
        q_query.advance_to(1);

        while probe.lt(&RootTimestamp::new(1)) {
            root.step();
        }
        root.step();

        println!("derivation:\t{}", time::precise_time_s() - start);
        let timer = time::precise_time_s();

        p_query.send(((36465u32,10135u32), 1));
        p_query.advance_to(2);
        q_query.advance_to(2);

        while probe.lt(&RootTimestamp::new(2)) {
            root.step();
        }

        println!("query:\t{}", time::precise_time_s() - timer);

        p_query.close();
        q_query.close();

        while root.step() { }    // wind down the computation

    });
}
<|endoftext|>


<commit_before>extern crate docopt;
extern crate flac;
extern crate rustc_serialize;

use docopt::Docopt;
use flac::{ReadStream, Stream, StreamProducer};
use flac::metadata;
use flac::metadata::VorbisComment;

use std::env;
use std::fs::File;

const USAGE: &'static str = "
Usage: metadata streaminfo [options] <input>
       metadata comments [options] <input>
       metadata --help

Options:
  --block-size       Show both the max and min block size from StreamInfo.
  --frame-size       Show both the max and min frame size from StreamInfo.
  --sample-rate      Show the sample rate from StreamInfo.
  --channels         Show the number of channels from StreamInfo.
  --bits-per-sample  Show the size in bits for each sample from StreamInfo.
  --total-samples    Show total number of samples from StreamInfo.
  --md5              Show the MD5 signature from StreamInfo.
  --vendor           Show the vendor string from VorbisComment.
  --name=NAME        Show the comments matching the `NAME` from VorbisComment.
  -h, --help         Show this message.
";

#[derive(Debug, RustcDecodable)]
struct Arguments {
  arg_input: String,
  cmd_streaminfo: bool,
  cmd_comments: bool,
  flag_block_size: bool,
  flag_frame_size: bool,
  flag_sample_rate: bool,
  flag_channels: bool,
  flag_bits_per_sample: bool,
  flag_total_samples: bool,
  flag_md5: bool,
  flag_vendor: bool,
  flag_name: Option<String>,
}

macro_rules! format_print (
  ($format_str: expr, $opt_str: expr, $data: expr, $no_flag: expr) => (
    {
      println!($format_str, if $no_flag {
        $opt_str
      } else {
        ""
      }, $data);
    }
  );
);

fn print_stream_info<P>(stream: &Stream<P>, args: &Arguments)
 where P: StreamProducer {
  let info     = stream.info();
  let no_flags = (args.flag_block_size      || args.flag_frame_size    ||
                  args.flag_sample_rate     || args.flag_channels      ||
                  args.flag_bits_per_sample || args.flag_total_samples ||
                  args.flag_md5) == false;

  if no_flags || args.flag_block_size {
    println!("Minimum block size: {} samples", info.min_block_size);
    println!("Maximum block size: {} samples", info.max_block_size);
  }

  if no_flags || args.flag_frame_size {
    println!("Minimum frame size: {} bytes", info.min_frame_size);
    println!("Maximum frame size: {} bytes", info.max_frame_size);
  }

  if no_flags || args.flag_sample_rate {
    format_print!("{}{} Hz", "Sample rate: ", info.sample_rate, no_flags);
  }

  if no_flags || args.flag_channels {
    format_print!("{}{}", "Number of channels: ", info.channels, no_flags);
  }

  if no_flags || args.flag_bits_per_sample {
    format_print!("{}{}", "Bits per samples: ", info.bits_per_sample,
                                                no_flags);
  }

  if no_flags || args.flag_total_samples {
    format_print!("{}{}", "Total samples: ", info.total_samples, no_flags);
  }

  if no_flags || args.flag_md5 {
    let mut md5  = String::with_capacity(32);

    for byte in &info.md5_sum {
      let hex = format!("{:02x}", byte);

      md5.push_str(&hex);
    }

    format_print!("{}{}", "MD5 sum: ", md5, no_flags);
  }
}

fn print_vorbis_comments(vorbis_comment: &VorbisComment, args: &Arguments) {
  let mut index = 1;
  let no_flags  = (args.flag_vendor || args.flag_name.is_some()) == false;

  if no_flags || args.flag_vendor {
    format_print!("{}{}", "Vendor string: ", vorbis_comment.vendor_string,
                                             no_flags);
  }

  if no_flags {
    println!("Number of Comments: {}", vorbis_comment.comments.len());
  }

  for comment in &vorbis_comment.comments {
    let is_name = if let Some(ref name) = args.flag_name {
      name == comment.0
    } else {
      false
    };

    if no_flags || is_name {
      println!("  {}: \"{}\" = {}", index, comment.0, comment.1);
    }

    index += 1;
  }
}

fn main() {
  let args: Arguments = Docopt::new(USAGE)
    .and_then(|d| d.argv(env::args()).decode())
    .unwrap_or_else(|e| e.exit());

  let stream = Stream::<ReadStream<File>>::from_file(&args.arg_input)
                 .expect("Couldn't parse file");

  if args.cmd_streaminfo {
    print_stream_info(&stream, &args);
  }

  for meta in stream.metadata() {
    match meta.data {
      metadata::Data::VorbisComment(ref v) => {
        if args.cmd_comments {
          print_vorbis_comments(v, &args)
        }
      }
      _                                    => continue,
    }
  }
}
<commit_msg>Change formating of comments when it is single<commit_after>extern crate docopt;
extern crate flac;
extern crate rustc_serialize;

use docopt::Docopt;
use flac::{ReadStream, Stream, StreamProducer};
use flac::metadata;
use flac::metadata::VorbisComment;

use std::env;
use std::fs::File;

const USAGE: &'static str = "
Usage: metadata streaminfo [options] <input>
       metadata comments [options] <input>
       metadata --help

Options:
  --block-size       Show both the max and min block size from StreamInfo.
  --frame-size       Show both the max and min frame size from StreamInfo.
  --sample-rate      Show the sample rate from StreamInfo.
  --channels         Show the number of channels from StreamInfo.
  --bits-per-sample  Show the size in bits for each sample from StreamInfo.
  --total-samples    Show total number of samples from StreamInfo.
  --md5              Show the MD5 signature from StreamInfo.
  --vendor           Show the vendor string from VorbisComment.
  --name=NAME        Show the comments matching the `NAME` from VorbisComment.
  -h, --help         Show this message.
";

#[derive(Debug, RustcDecodable)]
struct Arguments {
  arg_input: String,
  cmd_streaminfo: bool,
  cmd_comments: bool,
  flag_block_size: bool,
  flag_frame_size: bool,
  flag_sample_rate: bool,
  flag_channels: bool,
  flag_bits_per_sample: bool,
  flag_total_samples: bool,
  flag_md5: bool,
  flag_vendor: bool,
  flag_name: Option<String>,
}

macro_rules! format_print (
  ($format_str: expr, $opt_str: expr, $data: expr, $no_flag: expr) => (
    {
      println!($format_str, if $no_flag {
        $opt_str
      } else {
        ""
      }, $data);
    }
  );
);

fn print_stream_info<P>(stream: &Stream<P>, args: &Arguments)
 where P: StreamProducer {
  let info     = stream.info();
  let no_flags = (args.flag_block_size      || args.flag_frame_size    ||
                  args.flag_sample_rate     || args.flag_channels      ||
                  args.flag_bits_per_sample || args.flag_total_samples ||
                  args.flag_md5) == false;

  if no_flags || args.flag_block_size {
    println!("Minimum block size: {} samples", info.min_block_size);
    println!("Maximum block size: {} samples", info.max_block_size);
  }

  if no_flags || args.flag_frame_size {
    println!("Minimum frame size: {} bytes", info.min_frame_size);
    println!("Maximum frame size: {} bytes", info.max_frame_size);
  }

  if no_flags || args.flag_sample_rate {
    format_print!("{}{} Hz", "Sample rate: ", info.sample_rate, no_flags);
  }

  if no_flags || args.flag_channels {
    format_print!("{}{}", "Number of channels: ", info.channels, no_flags);
  }

  if no_flags || args.flag_bits_per_sample {
    format_print!("{}{}", "Bits per samples: ", info.bits_per_sample,
                                                no_flags);
  }

  if no_flags || args.flag_total_samples {
    format_print!("{}{}", "Total samples: ", info.total_samples, no_flags);
  }

  if no_flags || args.flag_md5 {
    let mut md5  = String::with_capacity(32);

    for byte in &info.md5_sum {
      let hex = format!("{:02x}", byte);

      md5.push_str(&hex);
    }

    format_print!("{}{}", "MD5 sum: ", md5, no_flags);
  }
}

fn print_vorbis_comments(vorbis_comment: &VorbisComment, args: &Arguments) {
  let no_flags  = (args.flag_vendor || args.flag_name.is_some()) == false;

  if no_flags || args.flag_vendor {
    format_print!("{}{}", "Vendor string: ", vorbis_comment.vendor_string,
                                             no_flags);
  }

  if no_flags {
    let mut index = 1;

    println!("Number of Comments: {}", vorbis_comment.comments.len());

    for (name, value) in &vorbis_comment.comments {
      println!("  {}: \"{}\" = {}", index, name, value);

      index += 1;
    }
  } else {
    if let Some(ref name) = args.flag_name {
      vorbis_comment.comments.get(name)
                             .map(|value| println!("{}", value));
    }
  }
}

fn main() {
  let args: Arguments = Docopt::new(USAGE)
    .and_then(|d| d.argv(env::args()).decode())
    .unwrap_or_else(|e| e.exit());

  let stream = Stream::<ReadStream<File>>::from_file(&args.arg_input)
                 .expect("Couldn't parse file");

  if args.cmd_streaminfo {
    print_stream_info(&stream, &args);
  }

  for meta in stream.metadata() {
    match meta.data {
      metadata::Data::VorbisComment(ref v) => {
        if args.cmd_comments {
          print_vorbis_comments(v, &args)
        }
      }
      _                                    => continue,
    }
  }
}
<|endoftext|>


<commit_before>//! Source code browser

use std::collections::BTreeMap;
use std::cmp::Ordering;
use super::MetaData;
use super::page::Page;
use super::pool::Pool;
use super::file::File as DbFile;
use iron::prelude::*;
use router::Router;
use rustc_serialize::json::{Json, ToJson};
use postgres::Connection;

#[derive(PartialEq, PartialOrd)]
enum FileType {
    Dir,
    Text,
    Binary,
    RustSource,
}

#[derive(PartialEq, PartialOrd)]
struct File {
    name: String,
    file_type: FileType,
}

struct FileList {
    metadata: MetaData,
    files: Vec<File>,
}

impl ToJson for FileList {
    fn to_json(&self) -> Json {
        let mut m: BTreeMap<String, Json> = BTreeMap::new();

        m.insert("metadata".to_string(), self.metadata.to_json());

        let mut file_vec: Vec<Json> = Vec::new();

        for file in &self.files {
            let mut file_m: BTreeMap<String, Json> = BTreeMap::new();
            file_m.insert("name".to_string(), file.name.to_json());

            file_m.insert(match file.file_type {
                              FileType::Dir => "file_type_dir".to_string(),
                              FileType::Text => "file_type_text".to_string(),
                              FileType::Binary => "file_type_binary".to_string(),
                              FileType::RustSource => "file_type_rust_source".to_string(),
                          },
                          true.to_json());

            file_vec.push(file_m.to_json());
        }

        m.insert("files".to_string(), file_vec.to_json());
        m.to_json()
    }
}

impl FileList {
    /// Gets FileList from a request path
    ///
    /// All paths stored in database have this format:
    ///
    /// ```text
    /// [
    ///   ["text/plain",".gitignore"],
    ///   ["text/x-c","src/reseeding.rs"],
    ///   ["text/x-c","src/lib.rs"],
    ///   ["text/x-c","README.md"],
    ///   ...
    /// ]
    /// ```
    ///
    /// This function is only returning FileList for requested directory. If is empty,
    /// it will return list of files (and dirs) for root directory. req_path must be a
    /// directory or empty for root directory.
    pub fn from_path(conn: &Connection,
                     name: &str,
                     version: &str,
                     req_path: &str)
                     -> Option<FileList> {

        let rows = conn.query("SELECT crates.name,
                                      releases.version,
                                      releases.description,
                                      releases.target_name,
                                      releases.rustdoc_status,
                                      releases.files
                               FROM releases
                               LEFT OUTER JOIN crates ON crates.id = releases.crate_id
                               WHERE crates.name = $1 AND releases.version = $2",
                   &[&name, &version])
            .unwrap();

        if rows.len() == 0 {
            return None;
        }

        let files: Json = rows.get(0).get(5);

        let mut file_list: Vec<File> = Vec::new();

        files.as_array().map(|files| {
            for file in files {
                file.as_array().map(|file| {
                    let mime = file[0].as_string().unwrap();
                    let path = file[1].as_string().unwrap();

                    // skip .cargo-ok generated by cargo
                    if path == ".cargo-ok" {
                        return;
                    }

                    // look only files for req_path
                    if path.starts_with(&req_path) {
                        // remove req_path from path to reach files in this directory
                        let path = path.replace(&req_path, "");
                        let path_splited: Vec<&str> = path.split("/").collect();

                        // if path have '/' it is a directory
                        let ftype = if path_splited.len() > 1 {
                            FileType::Dir
                        } else if mime.starts_with("text") &&
                                              path_splited[0].ends_with(".rs") {
                            FileType::RustSource
                        } else if mime.starts_with("text") {
                            FileType::Text
                        } else {
                            FileType::Binary
                        };

                        let file = File {
                            name: path_splited[0].to_owned(),
                            file_type: ftype,
                        };

                        // avoid adding duplicates, a directory may occur more than once
                        if !file_list.contains(&file) {
                            file_list.push(file);
                        }
                    }
                });
            }
        });

        if file_list.is_empty() {
            return None;
        }

        file_list.sort_by(|a, b| {
            // directories must be listed first
            if a.file_type == FileType::Dir && b.file_type != FileType::Dir {
                Ordering::Less
            } else if a.file_type != FileType::Dir && b.file_type == FileType::Dir {
                Ordering::Greater
            } else {
                a.name.to_lowercase().cmp(&b.name.to_lowercase())
            }
        });

        Some(FileList {
            metadata: MetaData {
                name: rows.get(0).get(0),
                version: rows.get(0).get(1),
                description: rows.get(0).get(2),
                target_name: rows.get(0).get(3),
                rustdoc_status: rows.get(0).get(4),
            },
            files: file_list,
        })
    }
}

pub fn source_browser_handler(req: &mut Request) -> IronResult<Response> {
    let router = extension!(req, Router);
    let name = cexpect!(router.find("name"));
    let version = cexpect!(router.find("version"));

    // get path (req_path) for FileList::from_path and actual path for super::file::File::from_path
    let (req_path, file_path) = {
        let mut req_path = req.url.path();
        // remove first elements from path which is /crate/:name/:version/source
        for _ in 0..4 {
            req_path.remove(0);
        }
        let file_path = format!("sources/{}/{}/{}", name, version, req_path.join("/"));

        // FileList::from_path is only working for directories
        // remove file name if it's not a directory
        req_path.last_mut().map(|last| {
            if !last.is_empty() {
                *last = "";
            }
        });

        // remove crate name and version from req_path
        let path = req_path.join("/").replace(&format!("{}/{}/", name, version), "");

        (path, file_path)
    };

    let conn = extension!(req, Pool);

    // try to get actual file first
    // skip if request is a directory
    let file = if !file_path.ends_with("/") {
        DbFile::from_path(&conn, &file_path)
    } else {
        None
    };

    let (content, is_rust_source) = if let Some(file) = file {
        // serve the file with DatabaseFileHandler if file isn't text and not empty
        if !file.0.mime.starts_with("text") && !file.is_empty() {
            return Ok(file.serve());
        } else if file.0.mime.starts_with("text") && !file.is_empty() {
            (String::from_utf8(file.0.content).ok(), file.0.path.ends_with(".rs"))
        } else {
            (None, false)
        }
    } else {
        (None, false)
    };

    let list = FileList::from_path(&conn, &name, &version, &req_path);

    let page = Page::new(list)
        .set_bool("show_parent_link", !req_path.is_empty())
        .set_true("javascript_highlightjs")
        .set_true("show_package_navigation")
        .set_true("package_source_tab");

    if let Some(content) = content {
        page.set("file_content", &content)
            .set_bool("file_content_rust_source", is_rust_source)
            .to_resp("source")
    } else {
        page.to_resp("source")
    }
}
<commit_msg>fix panic if crate failed to build<commit_after>//! Source code browser

use std::collections::BTreeMap;
use std::cmp::Ordering;
use super::MetaData;
use super::page::Page;
use super::pool::Pool;
use super::file::File as DbFile;
use iron::prelude::*;
use router::Router;
use rustc_serialize::json::{Json, ToJson};
use postgres::Connection;

#[derive(PartialEq, PartialOrd)]
enum FileType {
    Dir,
    Text,
    Binary,
    RustSource,
}

#[derive(PartialEq, PartialOrd)]
struct File {
    name: String,
    file_type: FileType,
}

struct FileList {
    metadata: MetaData,
    files: Vec<File>,
}

impl ToJson for FileList {
    fn to_json(&self) -> Json {
        let mut m: BTreeMap<String, Json> = BTreeMap::new();

        m.insert("metadata".to_string(), self.metadata.to_json());

        let mut file_vec: Vec<Json> = Vec::new();

        for file in &self.files {
            let mut file_m: BTreeMap<String, Json> = BTreeMap::new();
            file_m.insert("name".to_string(), file.name.to_json());

            file_m.insert(match file.file_type {
                              FileType::Dir => "file_type_dir".to_string(),
                              FileType::Text => "file_type_text".to_string(),
                              FileType::Binary => "file_type_binary".to_string(),
                              FileType::RustSource => "file_type_rust_source".to_string(),
                          },
                          true.to_json());

            file_vec.push(file_m.to_json());
        }

        m.insert("files".to_string(), file_vec.to_json());
        m.to_json()
    }
}

impl FileList {
    /// Gets FileList from a request path
    ///
    /// All paths stored in database have this format:
    ///
    /// ```text
    /// [
    ///   ["text/plain",".gitignore"],
    ///   ["text/x-c","src/reseeding.rs"],
    ///   ["text/x-c","src/lib.rs"],
    ///   ["text/x-c","README.md"],
    ///   ...
    /// ]
    /// ```
    ///
    /// This function is only returning FileList for requested directory. If is empty,
    /// it will return list of files (and dirs) for root directory. req_path must be a
    /// directory or empty for root directory.
    pub fn from_path(conn: &Connection,
                     name: &str,
                     version: &str,
                     req_path: &str)
                     -> Option<FileList> {

        let rows = conn.query("SELECT crates.name,
                                      releases.version,
                                      releases.description,
                                      releases.target_name,
                                      releases.rustdoc_status,
                                      releases.files
                               FROM releases
                               LEFT OUTER JOIN crates ON crates.id = releases.crate_id
                               WHERE crates.name = $1 AND releases.version = $2",
                   &[&name, &version])
            .unwrap();

        if rows.len() == 0 {
            return None;
        }

        let files: Json = rows.get(0).get_opt(5).unwrap().ok()?;

        let mut file_list: Vec<File> = Vec::new();

        files.as_array().map(|files| {
            for file in files {
                file.as_array().map(|file| {
                    let mime = file[0].as_string().unwrap();
                    let path = file[1].as_string().unwrap();

                    // skip .cargo-ok generated by cargo
                    if path == ".cargo-ok" {
                        return;
                    }

                    // look only files for req_path
                    if path.starts_with(&req_path) {
                        // remove req_path from path to reach files in this directory
                        let path = path.replace(&req_path, "");
                        let path_splited: Vec<&str> = path.split("/").collect();

                        // if path have '/' it is a directory
                        let ftype = if path_splited.len() > 1 {
                            FileType::Dir
                        } else if mime.starts_with("text") &&
                                              path_splited[0].ends_with(".rs") {
                            FileType::RustSource
                        } else if mime.starts_with("text") {
                            FileType::Text
                        } else {
                            FileType::Binary
                        };

                        let file = File {
                            name: path_splited[0].to_owned(),
                            file_type: ftype,
                        };

                        // avoid adding duplicates, a directory may occur more than once
                        if !file_list.contains(&file) {
                            file_list.push(file);
                        }
                    }
                });
            }
        });

        if file_list.is_empty() {
            return None;
        }

        file_list.sort_by(|a, b| {
            // directories must be listed first
            if a.file_type == FileType::Dir && b.file_type != FileType::Dir {
                Ordering::Less
            } else if a.file_type != FileType::Dir && b.file_type == FileType::Dir {
                Ordering::Greater
            } else {
                a.name.to_lowercase().cmp(&b.name.to_lowercase())
            }
        });

        Some(FileList {
            metadata: MetaData {
                name: rows.get(0).get(0),
                version: rows.get(0).get(1),
                description: rows.get(0).get(2),
                target_name: rows.get(0).get(3),
                rustdoc_status: rows.get(0).get(4),
            },
            files: file_list,
        })
    }
}

pub fn source_browser_handler(req: &mut Request) -> IronResult<Response> {
    use iron::status;
    use super::error::Nope;
    let router = extension!(req, Router);
    let name = cexpect!(router.find("name"));
    let version = cexpect!(router.find("version"));

    // get path (req_path) for FileList::from_path and actual path for super::file::File::from_path
    let (req_path, file_path) = {
        let mut req_path = req.url.path();
        // remove first elements from path which is /crate/:name/:version/source
        for _ in 0..4 {
            req_path.remove(0);
        }
        let file_path = format!("sources/{}/{}/{}", name, version, req_path.join("/"));

        // FileList::from_path is only working for directories
        // remove file name if it's not a directory
        req_path.last_mut().map(|last| {
            if !last.is_empty() {
                *last = "";
            }
        });

        // remove crate name and version from req_path
        let path = req_path.join("/").replace(&format!("{}/{}/", name, version), "");

        (path, file_path)
    };

    let conn = extension!(req, Pool);

    // try to get actual file first
    // skip if request is a directory
    let file = if !file_path.ends_with("/") {
        DbFile::from_path(&conn, &file_path)
    } else {
        None
    };

    let (content, is_rust_source) = if let Some(file) = file {
        // serve the file with DatabaseFileHandler if file isn't text and not empty
        if !file.0.mime.starts_with("text") && !file.is_empty() {
            return Ok(file.serve());
        } else if file.0.mime.starts_with("text") && !file.is_empty() {
            (String::from_utf8(file.0.content).ok(), file.0.path.ends_with(".rs"))
        } else {
            (None, false)
        }
    } else {
        (None, false)
    };

    let list = FileList::from_path(&conn, &name, &version, &req_path);

    let page = Page::new(list)
        .set_bool("show_parent_link", !req_path.is_empty())
        .set_true("javascript_highlightjs")
        .set_true("show_package_navigation")
        .set_true("package_source_tab");

    if let Some(content) = content {
        page.set("file_content", &content)
            .set_bool("file_content_rust_source", is_rust_source)
            .to_resp("source")
    } else {
        Err(IronError::new(Nope::CrateNotFound, status::NotFound))
    }
}
<|endoftext|>


<commit_before> #![feature(buf_stream)]

#[macro_use]
extern crate log;
extern crate url;
extern crate capnp;

pub mod yak_capnp;

use std::net::TcpStream;
use std::io::{self,BufStream,Write};
use capnp::serialize_packed;
use capnp::{MessageBuilder, MessageReader, MallocMessageBuilder, ReaderOptions};

use url::{Url,SchemeType,UrlParser};

use yak_capnp::{client_request,client_response};

fn yak_url_scheme(_scheme: &str) -> SchemeType {
  SchemeType::Relative(0)
}

pub struct Client {
  connection: BufStream<TcpStream>
}

#[derive(Debug)]
pub enum YakError {
  UrlParseError(url::ParseError),
  InvalidUrl(Url),
  IoError(io::Error),
  CapnpError(capnp::Error),
  CapnpNotInSchema(capnp::NotInSchema),
  ProtocolError
}

#[derive(PartialEq,Eq,PartialOrd,Ord,Debug)]
pub struct Datum {
  pub content: Vec<u8>
}

impl Client {
  pub fn connect(loc: &str) -> Result<Client, YakError> {
    let mut p = UrlParser::new();
    p.scheme_type_mapper(yak_url_scheme);
    let url = try!(p.parse(loc));
    debug!("yak:url: {:?}", url);
    let addr = match (url.domain(), url.port()) {
      (Some(host), Some(port)) => (host, port),
      _ => return Err(YakError::InvalidUrl(url.clone()))
    };

    let sock = try!(TcpStream::connect(addr));
    debug!("connected:{:?}", sock);
    Ok(Client { connection: BufStream::new(sock) })
  }

  fn encode_truncate<B: MessageBuilder>(message: &mut B) {
    let mut rec = message.init_root::<client_request::Builder>();
    rec.set_truncate(())
  }

  pub fn truncate(&mut self) -> Result<(), YakError> {
    let mut message = MallocMessageBuilder::new_default();
    Self::encode_truncate(&mut message);
    try!(serialize_packed::write_message(&mut self.connection, &mut message));
    try!(self.connection.flush());
    debug!("Waiting for response");

    let message_reader = try!(serialize_packed::read_message(&mut self.connection, ReaderOptions::new()));
    let msg = try!(message_reader.get_root::<client_response::Reader>());
    debug!("Got response");

    Ok(())
  }

  fn encode_write<B: MessageBuilder>(message: &mut B, key: &[u8], val: &[u8]) {
    let mut rec = message.init_root::<client_request::Builder>();
    let mut req = rec.init_write();
    req.set_key(key);
    req.set_value(val);
  }

  pub fn write(&mut self, key: &[u8], val: &[u8]) -> Result<(), YakError> {
    let mut message = MallocMessageBuilder::new_default();
    Self::encode_write(&mut message, key, val);
    try!(serialize_packed::write_message(&mut self.connection, &mut message));
    try!(self.connection.flush());
    debug!("Waiting for response");

    let message_reader = try!(serialize_packed::read_message(&mut self.connection, ReaderOptions::new()));
    let msg = try!(message_reader.get_root::<client_response::Reader>());
    debug!("Got response");

    Ok(())
  }

  fn encode_read<B: MessageBuilder>(message: &mut B, key: &[u8]) {
      let mut rec = message.init_root::<client_request::Builder>();
      let mut req = rec.init_read();
      req.set_key(key)
  }

  pub fn read(&mut self, key: &[u8]) -> Result<Vec<Datum>, YakError> {
    let mut message = MallocMessageBuilder::new_default();
    Self::encode_read(&mut message, key);
    try!(serialize_packed::write_message(&mut self.connection, &mut message));
    try!(self.connection.flush());
    debug!("Waiting for response");

    let message_reader = try!(serialize_packed::read_message(&mut self.connection, ReaderOptions::new()));
    let msg = try!(message_reader.get_root::<client_response::Reader>());
    match try!(msg.which()) {
      client_response::OkData(d) => {
        debug!("Got response Data: ");
        let mut data = Vec::with_capacity(try!(d).len() as usize);
        for it in try!(d).iter() {
          let val : Vec<u8> = try!(it.get_value()).iter().map(|v|v.clone()).collect();
          data.push(Datum { content: val });
        }

        Ok(data)
      },
      other => Err(YakError::ProtocolError)
    }

  }
}

impl From<url::ParseError> for YakError {
  fn from(err: url::ParseError) -> YakError {
    YakError::UrlParseError(err)
  }
}

impl From<io::Error> for YakError {
  fn from(err: io::Error) -> YakError {
    YakError::IoError(err)
  }
}

impl From<capnp::Error> for YakError {
  fn from(err: capnp::Error) -> YakError {
    YakError::CapnpError(err)
  }
}

impl From<capnp::NotInSchema> for YakError {
  fn from(err: capnp::NotInSchema) -> YakError {
    YakError::CapnpNotInSchema(err)
  }
}
<commit_msg>Extract functions to handle response.<commit_after> #![feature(buf_stream)]

#[macro_use]
extern crate log;
extern crate url;
extern crate capnp;

pub mod yak_capnp;

use std::net::TcpStream;
use std::io::{self,BufStream,Write};
use capnp::serialize_packed;
use capnp::{MessageBuilder, MessageReader, MallocMessageBuilder, ReaderOptions};

use url::{Url,SchemeType,UrlParser};

use yak_capnp::{client_request,client_response};

fn yak_url_scheme(_scheme: &str) -> SchemeType {
  SchemeType::Relative(0)
}

pub struct Client {
  connection: BufStream<TcpStream>
}

#[derive(Debug)]
pub enum YakError {
  UrlParseError(url::ParseError),
  InvalidUrl(Url),
  IoError(io::Error),
  CapnpError(capnp::Error),
  CapnpNotInSchema(capnp::NotInSchema),
  ProtocolError
}

#[derive(PartialEq,Eq,PartialOrd,Ord,Debug)]
pub struct Datum {
  pub content: Vec<u8>
}

impl Client {
  pub fn connect(loc: &str) -> Result<Client, YakError> {
    let mut p = UrlParser::new();
    p.scheme_type_mapper(yak_url_scheme);
    let url = try!(p.parse(loc));
    debug!("yak:url: {:?}", url);
    let addr = match (url.domain(), url.port()) {
      (Some(host), Some(port)) => (host, port),
      _ => return Err(YakError::InvalidUrl(url.clone()))
    };

    let sock = try!(TcpStream::connect(addr));
    debug!("connected:{:?}", sock);
    Ok(Client { connection: BufStream::new(sock) })
  }

  fn encode_truncate<B: MessageBuilder>(message: &mut B) {
    let mut rec = message.init_root::<client_request::Builder>();
    rec.set_truncate(())
  }

  fn encode_write<B: MessageBuilder>(message: &mut B, key: &[u8], val: &[u8]) {
    let mut rec = message.init_root::<client_request::Builder>();
    let mut req = rec.init_write();
    req.set_key(key);
    req.set_value(val);
  }

  fn encode_read<B: MessageBuilder>(message: &mut B, key: &[u8]) {
      let mut rec = message.init_root::<client_request::Builder>();
      let mut req = rec.init_read();
      req.set_key(key)
  }

  fn expect_ok<R: MessageReader>(message: &R) -> Result<(), YakError> {
    let msg = try!(message.get_root::<client_response::Reader>());
    debug!("Got response");
    match try!(msg.which()) {
      client_response::Ok(()) => Ok(()),
      other => Err(YakError::ProtocolError)
    }
  }

  fn expect_datum_list<R: MessageReader>(message_reader: &R) -> Result<Vec<Datum>, YakError> {
    let msg = try!(message_reader.get_root::<client_response::Reader>());
    match try!(msg.which()) {
      client_response::OkData(d) => {
        debug!("Got response Data: ");
        let mut data = Vec::with_capacity(try!(d).len() as usize);
        for it in try!(d).iter() {
          let val : Vec<u8> = try!(it.get_value()).iter().map(|v|v.clone()).collect();
          data.push(Datum { content: val });
        }

        Ok(data)
      },
      other => Err(YakError::ProtocolError)
    }
  }

  pub fn truncate(&mut self) -> Result<(), YakError> {
    let mut message = MallocMessageBuilder::new_default();
    Self::encode_truncate(&mut message);
    try!(serialize_packed::write_message(&mut self.connection, &mut message));
    try!(self.connection.flush());
    debug!("Waiting for response");

    let message_reader = try!(serialize_packed::read_message(&mut self.connection, ReaderOptions::new()));
    Self::expect_ok(&message_reader)
  }

  pub fn write(&mut self, key: &[u8], val: &[u8]) -> Result<(), YakError> {
    let mut message = MallocMessageBuilder::new_default();
    Self::encode_write(&mut message, key, val);
    try!(serialize_packed::write_message(&mut self.connection, &mut message));
    try!(self.connection.flush());
    debug!("Waiting for response");

    let message_reader = try!(serialize_packed::read_message(&mut self.connection, ReaderOptions::new()));
    Self::expect_ok(&message_reader)
  }

  pub fn read(&mut self, key: &[u8]) -> Result<Vec<Datum>, YakError> {
    let mut message = MallocMessageBuilder::new_default();
    Self::encode_read(&mut message, key);
    try!(serialize_packed::write_message(&mut self.connection, &mut message));
    try!(self.connection.flush());
    debug!("Waiting for response");

    let message_reader = try!(serialize_packed::read_message(&mut self.connection, ReaderOptions::new()));
    Self::expect_datum_list(&message_reader)
  }
}

impl From<url::ParseError> for YakError {
  fn from(err: url::ParseError) -> YakError {
    YakError::UrlParseError(err)
  }
}

impl From<io::Error> for YakError {
  fn from(err: io::Error) -> YakError {
    YakError::IoError(err)
  }
}

impl From<capnp::Error> for YakError {
  fn from(err: capnp::Error) -> YakError {
    YakError::CapnpError(err)
  }
}

impl From<capnp::NotInSchema> for YakError {
  fn from(err: capnp::NotInSchema) -> YakError {
    YakError::CapnpNotInSchema(err)
  }
}
<|endoftext|>


<commit_before><commit_msg>export example<commit_after><|endoftext|>


<commit_before>use std::io;

use tag;
use err;

#[macro_export]
macro_rules! asn1_info {
  ($rs_type:ty, $asn1_ty:expr, $class:expr, $tagnum:expr, $constructed:expr) => (
    impl $crate::serial::traits::Asn1Info for $rs_type {
      fn asn1_type() -> tag::Type {
        tag::Type::from($asn1_ty)
      }

      fn asn1_class() -> tag::Class {
        tag::Class::from($class)
      }

      fn asn1_tagnum() -> tag::TagNum {
        tag::TagNum::from($tagnum)
      }

      fn asn1_constructed() -> bool {
        $constructed
      }
    }
  )
}

#[macro_export]
macro_rules! asn1_sequence_info {
  ($rs_type:ty, $asn1_ty:expr) => (
    impl $crate::serial::traits::Asn1Info for $rs_type {
      fn asn1_type(&self) -> tag::Type {
        $asn1_ty.into()
      }

      fn asn1_class(&self) -> tag::Class {
        tag::Class::Universal
      }

      fn asn1_tagnum(&self) -> tag::TagNum {
        tag::TagNum::from(0x10 as u8)
      }

      fn asn1_constructed(&self) -> bool {
        true
      }
    }
  )
}

#[macro_export]
macro_rules! asn1_sequence_serialize {
  ($rs_type:ty, $($item:ident),*) => (
    impl serial::traits::Asn1Serialize for $rs_type {
      fn serialize_bytes<W: io::Write>(&self, writer: &mut W) -> Result<(), err::EncodeError> {
        // For each declared sequence member, serialize it onto the stream.
        $(
          try!(
            serial::traits::Asn1Serialize::serialize(&self.$item, writer)
          );
        )*
        Ok(())
      }
    }
  )
}

#[macro_export]
macro_rules! asn1_sequence_deserialize {
  ($rs_type:ty) => (
    impl serial::traits::Asn1Deserialize for u64 {
      fn deserialize<I: Iterator<Item=io::Result<u8>>>(reader: I) -> Result<Self, err::DecodeError> {
        unimplemented!();
      }
    }
  )
}
<commit_msg>Add context-specific tags to sequences<commit_after>use std::io;

use tag;
use err;

#[macro_export]
macro_rules! asn1_info {
  ($rs_type:ty, $asn1_ty:expr, $class:expr, $tagnum:expr, $constructed:expr) => (
    impl $crate::serial::traits::Asn1Info for $rs_type {
      fn asn1_type() -> tag::Type {
        tag::Type::from($asn1_ty)
      }

      fn asn1_class() -> tag::Class {
        tag::Class::from($class)
      }

      fn asn1_tagnum() -> tag::TagNum {
        tag::TagNum::from($tagnum)
      }

      fn asn1_constructed() -> bool {
        $constructed
      }
    }
  )
}

#[macro_export]
macro_rules! asn1_sequence_info {
  ($rs_type:ty, $asn1_ty:expr) => (
    impl $crate::serial::traits::Asn1Info for $rs_type {
      fn asn1_type(&self) -> tag::Type {
        $asn1_ty.into()
      }

      fn asn1_class(&self) -> tag::Class {
        tag::Class::Universal
      }

      fn asn1_tagnum(&self) -> tag::TagNum {
        tag::TagNum::from(0x10 as u8)
      }

      fn asn1_constructed(&self) -> bool {
        true
      }
    }
  )
}

#[macro_export]
macro_rules! asn1_sequence_serialize {
  ($rs_type:ty, $($item:ident),*) => (
    impl serial::traits::Asn1Serialize for $rs_type {
      fn serialize_bytes<W: io::Write>(&self, writer: &mut W) -> Result<(), err::EncodeError> {
        let mut bytes = Vec::new();
        let mut count: u64 = 0;
        // For each declared sequence member, serialize it onto the stream.
        $(
          count += 1;
          try!(
            serial::traits::Asn1Serialize::serialize(&self.$item, &mut bytes)
          );
          let tag = tag::Tag {
            class: tag::Class::ContextSpecific,
            tagnum: count.into(),
            len: Some(bytes.len() as tag::LenNum).into(),
            constructed: true,
          };
          try!(tag.encode_tag(writer));
          try!(writer.write(&mut bytes));

          bytes.clear();
        )*
        Ok(())
      }
    }
  )
}

#[macro_export]
macro_rules! asn1_sequence_deserialize {
  ($rs_type:ty) => (
    impl serial::traits::Asn1Deserialize for u64 {
      fn deserialize<I: Iterator<Item=io::Result<u8>>>(reader: I) -> Result<Self, err::DecodeError> {
        unimplemented!();
      }
    }
  )
}
<|endoftext|>


<commit_before>/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0.
 */

// snippet-start:[rust.example_code.s3.large_files.scenario]

use std::convert::TryInto;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;

use aws_config::meta::region::RegionProviderChain;
use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart};
use aws_sdk_s3::output::{CreateMultipartUploadOutput, GetObjectOutput};
use aws_sdk_s3::{Client as S3Client, Error, Region};
use aws_smithy_http::byte_stream::{ByteStream, Length};
use rand::distributions::Alphanumeric;
use rand::{thread_rng, Rng};
use uuid::Uuid;

//In bytes, minimum chunk size of 5MB. Increase CHUNK_SIZE to send larger chunks.
const CHUNK_SIZE: u64 = 1024 * 1024 * 5;
const MAX_CHUNKS: u64 = 10000;

#[tokio::main]
pub async fn main() -> Result<(), Error> {
    let shared_config = aws_config::load_from_env().await;
    let client = S3Client::new(&shared_config);

    let bucket_name = format!("doc-example-bucket-{}", Uuid::new_v4());
    let region_provider = RegionProviderChain::first_try(Region::new("us-west-2"));
    let region = region_provider.region().await.unwrap();
    s3_service::create_bucket(&client, &bucket_name, region.as_ref()).await?;

    let key = "sample.txt".to_string();
    // snippet-start:[rust.example_code.s3.create_multipart_upload]
    let multipart_upload_res: CreateMultipartUploadOutput = client
        .create_multipart_upload()
        .bucket(&bucket_name)
        .key(&key)
        .send()
        .await
        .unwrap();
    // snippet-end:[rust.example_code.s3.create_multipart_upload]
    let upload_id = multipart_upload_res.upload_id().unwrap();

    //Create a file of random characters for the upload.
    let mut file = File::create(&key).expect("Could not create sample file.");
    // Loop until the file is 5 chunks.
    while file.metadata().unwrap().len() <= CHUNK_SIZE * 4 {
        let rand_string: String = thread_rng()
            .sample_iter(&Alphanumeric)
            .take(256)
            .map(char::from)
            .collect();
        let return_string: String = "\n".to_string();
        file.write_all(rand_string.as_ref())
            .expect("Error writing to file.");
        file.write_all(return_string.as_ref())
            .expect("Error writing to file.");
    }

    let path = Path::new(&key);
    let file_size = tokio::fs::metadata(path)
        .await
        .expect("it exists I swear")
        .len();

    let mut chunk_count = (file_size / CHUNK_SIZE) + 1;
    let mut size_of_last_chunk = file_size % CHUNK_SIZE;
    if size_of_last_chunk == 0 {
        size_of_last_chunk = CHUNK_SIZE;
        chunk_count -= 1;
    }

    if file_size == 0 {
        panic!("Bad file size.");
    }
    if chunk_count > MAX_CHUNKS {
        panic!("Too many chunks! Try increasing your chunk size.")
    }

    let mut upload_parts: Vec<CompletedPart> = Vec::new();

    for chunk_index in 0..chunk_count {
        let this_chunk = if chunk_count - 1 == chunk_index {
            size_of_last_chunk
        } else {
            CHUNK_SIZE
        };
        let stream = ByteStream::read_from()
            .path(path)
            .offset(chunk_index * CHUNK_SIZE)
            .length(Length::Exact(this_chunk))
            .build()
            .await
            .unwrap();
        //Chunk index needs to start at 0, but part numbers start at 1.
        let part_number = (chunk_index as i32) + 1;
        // snippet-start:[rust.example_code.s3.upload_part]
        let upload_part_res = client
            .upload_part()
            .key(&key)
            .bucket(&bucket_name)
            .upload_id(upload_id)
            .body(stream)
            .part_number(part_number)
            .send()
            .await?;
        upload_parts.push(
            CompletedPart::builder()
                .e_tag(upload_part_res.e_tag.unwrap_or_default())
                .part_number(part_number)
                .build(),
        );
        // snippet-end:[rust.example_code.s3.upload_part]
    }
    // snippet-start:[rust.example_code.s3.upload_part.CompletedMultipartUpload]
    let completed_multipart_upload: CompletedMultipartUpload = CompletedMultipartUpload::builder()
        .set_parts(Some(upload_parts))
        .build();
    // snippet-end:[rust.example_code.s3.upload_part.CompletedMultipartUpload]

    // snippet-start:[rust.example_code.s3.complete_multipart_upload]
    let _complete_multipart_upload_res = client
        .complete_multipart_upload()
        .bucket(&bucket_name)
        .key(&key)
        .multipart_upload(completed_multipart_upload)
        .upload_id(upload_id)
        .send()
        .await.unwrap();
    // snippet-end:[rust.example_code.s3.complete_multipart_upload]

    let data: GetObjectOutput = s3_service::download_object(&client, &bucket_name, &key).await;
    let data_length: u64 = data.content_length().try_into().unwrap();
    if file.metadata().unwrap().len() == data_length {
        println!("Data lengths match.");
    } else {
        println!("The data was not the same size!");
    }

    s3_service::delete_objects(&client, &bucket_name)
        .await
        .expect("Error emptying bucket.");
    s3_service::delete_bucket(&client, &bucket_name)
        .await
        .expect("Error deleting bucket.");

    Ok(())
}
// snippet-end:[rust.example_code.s3.large_files.scenario]
<commit_msg>Formatting issue.<commit_after>/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0.
 */

// snippet-start:[rust.example_code.s3.large_files.scenario]

use std::convert::TryInto;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;

use aws_config::meta::region::RegionProviderChain;
use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart};
use aws_sdk_s3::output::{CreateMultipartUploadOutput, GetObjectOutput};
use aws_sdk_s3::{Client as S3Client, Error, Region};
use aws_smithy_http::byte_stream::{ByteStream, Length};
use rand::distributions::Alphanumeric;
use rand::{thread_rng, Rng};
use uuid::Uuid;

//In bytes, minimum chunk size of 5MB. Increase CHUNK_SIZE to send larger chunks.
const CHUNK_SIZE: u64 = 1024 * 1024 * 5;
const MAX_CHUNKS: u64 = 10000;

#[tokio::main]
pub async fn main() -> Result<(), Error> {
    let shared_config = aws_config::load_from_env().await;
    let client = S3Client::new(&shared_config);

    let bucket_name = format!("doc-example-bucket-{}", Uuid::new_v4());
    let region_provider = RegionProviderChain::first_try(Region::new("us-west-2"));
    let region = region_provider.region().await.unwrap();
    s3_service::create_bucket(&client, &bucket_name, region.as_ref()).await?;

    let key = "sample.txt".to_string();
    // snippet-start:[rust.example_code.s3.create_multipart_upload]
    let multipart_upload_res: CreateMultipartUploadOutput = client
        .create_multipart_upload()
        .bucket(&bucket_name)
        .key(&key)
        .send()
        .await
        .unwrap();
    // snippet-end:[rust.example_code.s3.create_multipart_upload]
    let upload_id = multipart_upload_res.upload_id().unwrap();

    //Create a file of random characters for the upload.
    let mut file = File::create(&key).expect("Could not create sample file.");
    // Loop until the file is 5 chunks.
    while file.metadata().unwrap().len() <= CHUNK_SIZE * 4 {
        let rand_string: String = thread_rng()
            .sample_iter(&Alphanumeric)
            .take(256)
            .map(char::from)
            .collect();
        let return_string: String = "\n".to_string();
        file.write_all(rand_string.as_ref())
            .expect("Error writing to file.");
        file.write_all(return_string.as_ref())
            .expect("Error writing to file.");
    }

    let path = Path::new(&key);
    let file_size = tokio::fs::metadata(path)
        .await
        .expect("it exists I swear")
        .len();

    let mut chunk_count = (file_size / CHUNK_SIZE) + 1;
    let mut size_of_last_chunk = file_size % CHUNK_SIZE;
    if size_of_last_chunk == 0 {
        size_of_last_chunk = CHUNK_SIZE;
        chunk_count -= 1;
    }

    if file_size == 0 {
        panic!("Bad file size.");
    }
    if chunk_count > MAX_CHUNKS {
        panic!("Too many chunks! Try increasing your chunk size.")
    }

    let mut upload_parts: Vec<CompletedPart> = Vec::new();

    for chunk_index in 0..chunk_count {
        let this_chunk = if chunk_count - 1 == chunk_index {
            size_of_last_chunk
        } else {
            CHUNK_SIZE
        };
        let stream = ByteStream::read_from()
            .path(path)
            .offset(chunk_index * CHUNK_SIZE)
            .length(Length::Exact(this_chunk))
            .build()
            .await
            .unwrap();
        //Chunk index needs to start at 0, but part numbers start at 1.
        let part_number = (chunk_index as i32) + 1;
        // snippet-start:[rust.example_code.s3.upload_part]
        let upload_part_res = client
            .upload_part()
            .key(&key)
            .bucket(&bucket_name)
            .upload_id(upload_id)
            .body(stream)
            .part_number(part_number)
            .send()
            .await?;
        upload_parts.push(
            CompletedPart::builder()
                .e_tag(upload_part_res.e_tag.unwrap_or_default())
                .part_number(part_number)
                .build(),
        );
        // snippet-end:[rust.example_code.s3.upload_part]
    }
    // snippet-start:[rust.example_code.s3.upload_part.CompletedMultipartUpload]
    let completed_multipart_upload: CompletedMultipartUpload = CompletedMultipartUpload::builder()
        .set_parts(Some(upload_parts))
        .build();
    // snippet-end:[rust.example_code.s3.upload_part.CompletedMultipartUpload]

    // snippet-start:[rust.example_code.s3.complete_multipart_upload]
    let _complete_multipart_upload_res = client
        .complete_multipart_upload()
        .bucket(&bucket_name)
        .key(&key)
        .multipart_upload(completed_multipart_upload)
        .upload_id(upload_id)
        .send()
        .await
        .unwrap();
    // snippet-end:[rust.example_code.s3.complete_multipart_upload]

    let data: GetObjectOutput = s3_service::download_object(&client, &bucket_name, &key).await;
    let data_length: u64 = data.content_length().try_into().unwrap();
    if file.metadata().unwrap().len() == data_length {
        println!("Data lengths match.");
    } else {
        println!("The data was not the same size!");
    }

    s3_service::delete_objects(&client, &bucket_name)
        .await
        .expect("Error emptying bucket.");
    s3_service::delete_bucket(&client, &bucket_name)
        .await
        .expect("Error deleting bucket.");

    Ok(())
}
// snippet-end:[rust.example_code.s3.large_files.scenario]
<|endoftext|>


<commit_before>use cairo::{self, ImageSurface};
use markup5ever::{expanded_name, local_name, namespace_url, ns, QualName};

use crate::drawing_ctx::DrawingCtx;
use crate::error::*;
use crate::node::{NodeResult, NodeTrait, RsvgNode};
use crate::parsers;
use crate::property_bag::PropertyBag;
use crate::surface_utils::{iterators::Pixels, shared_surface::SharedImageSurface};

use super::context::{FilterContext, FilterOutput, FilterResult};
use super::{FilterEffect, FilterError, Input, PrimitiveWithInput};

/// Enumeration of the color channels the displacement map can source.
#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
enum ColorChannel {
    R,
    G,
    B,
    A,
}

/// The `feDisplacementMap` filter primitive.
pub struct FeDisplacementMap {
    base: PrimitiveWithInput,
    in2: Option<Input>,
    scale: f64,
    x_channel_selector: ColorChannel,
    y_channel_selector: ColorChannel,
}

impl Default for FeDisplacementMap {
    /// Constructs a new `DisplacementMap` with empty properties.
    #[inline]
    fn default() -> FeDisplacementMap {
        FeDisplacementMap {
            base: PrimitiveWithInput::new::<Self>(),
            in2: None,
            scale: 0.0,
            x_channel_selector: ColorChannel::A,
            y_channel_selector: ColorChannel::A,
        }
    }
}

impl NodeTrait for FeDisplacementMap {
    impl_node_as_filter_effect!();

    fn set_atts(&mut self, parent: Option<&RsvgNode>, pbag: &PropertyBag<'_>) -> NodeResult {
        self.base.set_atts(parent, pbag)?;

        for (attr, value) in pbag.iter() {
            match attr.expanded() {
                expanded_name!(svg "in2") => self.in2 = Some(Input::parse(attr, value)?),
                expanded_name!(svg "scale") => self.scale = parsers::number(value).attribute(attr)?,
                expanded_name!(svg "xChannelSelector") => {
                    self.x_channel_selector = ColorChannel::parse(attr, value)?
                }
                expanded_name!(svg "yChannelSelector") => {
                    self.y_channel_selector = ColorChannel::parse(attr, value)?
                }
                _ => (),
            }
        }

        Ok(())
    }
}

impl FilterEffect for FeDisplacementMap {
    fn render(
        &self,
        _node: &RsvgNode,
        ctx: &FilterContext,
        draw_ctx: &mut DrawingCtx,
    ) -> Result<FilterResult, FilterError> {
        let input = self.base.get_input(ctx, draw_ctx)?;
        let displacement_input = ctx.get_input(draw_ctx, self.in2.as_ref())?;
        let bounds = self
            .base
            .get_bounds(ctx)
            .add_input(&input)
            .add_input(&displacement_input)
            .into_irect(draw_ctx);

        // Displacement map's values need to be non-premultiplied.
        let displacement_surface = displacement_input.surface().unpremultiply(bounds)?;

        let (sx, sy) = ctx.paffine().transform_distance(self.scale, self.scale);

        let output_surface = ImageSurface::create(
            cairo::Format::ARgb32,
            ctx.source_graphic().width(),
            ctx.source_graphic().height(),
        )?;

        {
            let cr = cairo::Context::new(&output_surface);

            for (x, y, displacement_pixel) in Pixels::new(&displacement_surface, bounds) {
                let get_value = |channel| match channel {
                    ColorChannel::R => displacement_pixel.r,
                    ColorChannel::G => displacement_pixel.g,
                    ColorChannel::B => displacement_pixel.b,
                    ColorChannel::A => displacement_pixel.a,
                };

                let process = |x| f64::from(x) / 255.0 - 0.5;

                let dx = process(get_value(self.x_channel_selector));
                let dy = process(get_value(self.y_channel_selector));

                let x = f64::from(x);
                let y = f64::from(y);
                let ox = sx * dx;
                let oy = sy * dy;

                // Doing this in a loop doesn't look too bad performance wise, and allows not to
                // manually implement bilinear or other interpolation.
                cr.rectangle(x, y, 1.0, 1.0);
                cr.reset_clip();
                cr.clip();

                input.surface().set_as_source_surface(&cr, -ox, -oy);
                cr.paint();
            }
        }

        Ok(FilterResult {
            name: self.base.result.clone(),
            output: FilterOutput {
                surface: SharedImageSurface::new(output_surface, input.surface().surface_type())?,
                bounds,
            },
        })
    }

    #[inline]
    fn is_affected_by_color_interpolation_filters(&self) -> bool {
        // Performance TODO: this converts in back and forth to linear RGB while technically it's
        // only needed for in2.
        true
    }
}

impl ColorChannel {
    fn parse(attr: QualName, s: &str) -> Result<Self, NodeError> {
        match s {
            "R" => Ok(ColorChannel::R),
            "G" => Ok(ColorChannel::G),
            "B" => Ok(ColorChannel::B),
            "A" => Ok(ColorChannel::A),
            _ => Err(ValueErrorKind::parse_error("invalid value")).attribute(attr),
        }
    }
}
<commit_msg>displacement_map.rs: impl Parse for ColorChannel instead of using a custom method<commit_after>use cairo::{self, ImageSurface};
use cssparser::Parser;
use markup5ever::{expanded_name, local_name, namespace_url, ns};

use crate::drawing_ctx::DrawingCtx;
use crate::error::*;
use crate::node::{NodeResult, NodeTrait, RsvgNode};
use crate::parsers::{self, Parse, ParseValue};
use crate::property_bag::PropertyBag;
use crate::surface_utils::{iterators::Pixels, shared_surface::SharedImageSurface};

use super::context::{FilterContext, FilterOutput, FilterResult};
use super::{FilterEffect, FilterError, Input, PrimitiveWithInput};

/// Enumeration of the color channels the displacement map can source.
#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
enum ColorChannel {
    R,
    G,
    B,
    A,
}

/// The `feDisplacementMap` filter primitive.
pub struct FeDisplacementMap {
    base: PrimitiveWithInput,
    in2: Option<Input>,
    scale: f64,
    x_channel_selector: ColorChannel,
    y_channel_selector: ColorChannel,
}

impl Default for FeDisplacementMap {
    /// Constructs a new `DisplacementMap` with empty properties.
    #[inline]
    fn default() -> FeDisplacementMap {
        FeDisplacementMap {
            base: PrimitiveWithInput::new::<Self>(),
            in2: None,
            scale: 0.0,
            x_channel_selector: ColorChannel::A,
            y_channel_selector: ColorChannel::A,
        }
    }
}

impl NodeTrait for FeDisplacementMap {
    impl_node_as_filter_effect!();

    fn set_atts(&mut self, parent: Option<&RsvgNode>, pbag: &PropertyBag<'_>) -> NodeResult {
        self.base.set_atts(parent, pbag)?;

        for (attr, value) in pbag.iter() {
            match attr.expanded() {
                expanded_name!(svg "in2") => self.in2 = Some(Input::parse(attr, value)?),
                expanded_name!(svg "scale") => {
                    self.scale = parsers::number(value).attribute(attr)?
                }
                expanded_name!(svg "xChannelSelector") => {
                    self.x_channel_selector = attr.parse(value)?
                }
                expanded_name!(svg "yChannelSelector") => {
                    self.y_channel_selector = attr.parse(value)?
                }
                _ => (),
            }
        }

        Ok(())
    }
}

impl FilterEffect for FeDisplacementMap {
    fn render(
        &self,
        _node: &RsvgNode,
        ctx: &FilterContext,
        draw_ctx: &mut DrawingCtx,
    ) -> Result<FilterResult, FilterError> {
        let input = self.base.get_input(ctx, draw_ctx)?;
        let displacement_input = ctx.get_input(draw_ctx, self.in2.as_ref())?;
        let bounds = self
            .base
            .get_bounds(ctx)
            .add_input(&input)
            .add_input(&displacement_input)
            .into_irect(draw_ctx);

        // Displacement map's values need to be non-premultiplied.
        let displacement_surface = displacement_input.surface().unpremultiply(bounds)?;

        let (sx, sy) = ctx.paffine().transform_distance(self.scale, self.scale);

        let output_surface = ImageSurface::create(
            cairo::Format::ARgb32,
            ctx.source_graphic().width(),
            ctx.source_graphic().height(),
        )?;

        {
            let cr = cairo::Context::new(&output_surface);

            for (x, y, displacement_pixel) in Pixels::new(&displacement_surface, bounds) {
                let get_value = |channel| match channel {
                    ColorChannel::R => displacement_pixel.r,
                    ColorChannel::G => displacement_pixel.g,
                    ColorChannel::B => displacement_pixel.b,
                    ColorChannel::A => displacement_pixel.a,
                };

                let process = |x| f64::from(x) / 255.0 - 0.5;

                let dx = process(get_value(self.x_channel_selector));
                let dy = process(get_value(self.y_channel_selector));

                let x = f64::from(x);
                let y = f64::from(y);
                let ox = sx * dx;
                let oy = sy * dy;

                // Doing this in a loop doesn't look too bad performance wise, and allows not to
                // manually implement bilinear or other interpolation.
                cr.rectangle(x, y, 1.0, 1.0);
                cr.reset_clip();
                cr.clip();

                input.surface().set_as_source_surface(&cr, -ox, -oy);
                cr.paint();
            }
        }

        Ok(FilterResult {
            name: self.base.result.clone(),
            output: FilterOutput {
                surface: SharedImageSurface::new(output_surface, input.surface().surface_type())?,
                bounds,
            },
        })
    }

    #[inline]
    fn is_affected_by_color_interpolation_filters(&self) -> bool {
        // Performance TODO: this converts in back and forth to linear RGB while technically it's
        // only needed for in2.
        true
    }
}

impl Parse for ColorChannel {
    fn parse<'i>(parser: &mut Parser<'i, '_>) -> Result<ColorChannel, ValueErrorKind> {
        parse_identifiers!(
            parser,
            "R" => ColorChannel::R,
            "G" => ColorChannel::G,
            "B" => ColorChannel::B,
            "A" => ColorChannel::A,
        )
        .map_err(|_: ParseError| ValueErrorKind::parse_error("parse error"))
    }
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

//! The type_coercion optimizer rule ensures that all operators are operating on
//! compatible types by adding explicit cast operations to expressions. For example,
//! the operation `c_float + c_int` would be rewritten as `c_float + CAST(c_int AS
//! float)`. This keeps the runtime query execution code much simpler.

use std::collections::HashMap;

use arrow::datatypes::Schema;

use crate::error::{ExecutionError, Result};
use crate::execution::physical_plan::udf::ScalarFunction;
use crate::logicalplan::LogicalPlan;
use crate::logicalplan::{Expr, LogicalPlanBuilder};
use crate::optimizer::optimizer::OptimizerRule;
use crate::optimizer::utils;

/// Implementation of type coercion optimizer rule
pub struct TypeCoercionRule<'a> {
    scalar_functions: &'a HashMap<String, Box<ScalarFunction>>,
}

impl<'a> TypeCoercionRule<'a> {
    /// Create a new type coercion optimizer rule using meta-data about registered
    /// scalar functions
    pub fn new(scalar_functions: &'a HashMap<String, Box<ScalarFunction>>) -> Self {
        Self { scalar_functions }
    }

    /// Rewrite an expression list to include explicit CAST operations when required
    fn rewrite_expr_list(&self, expr: &[Expr], schema: &Schema) -> Result<Vec<Expr>> {
        Ok(expr
            .iter()
            .map(|e| self.rewrite_expr(e, schema))
            .collect::<Result<Vec<_>>>()?)
    }

    /// Rewrite an expression to include explicit CAST operations when required
    fn rewrite_expr(&self, expr: &Expr, schema: &Schema) -> Result<Expr> {
        match expr {
            Expr::BinaryExpr { left, op, right } => {
                let left = self.rewrite_expr(left, schema)?;
                let right = self.rewrite_expr(right, schema)?;
                let left_type = left.get_type(schema)?;
                let right_type = right.get_type(schema)?;
                if left_type == right_type {
                    Ok(Expr::BinaryExpr {
                        left: Box::new(left),
                        op: op.clone(),
                        right: Box::new(right),
                    })
                } else {
                    let super_type = utils::get_supertype(&left_type, &right_type)?;
                    Ok(Expr::BinaryExpr {
                        left: Box::new(left.cast_to(&super_type, schema)?),
                        op: op.clone(),
                        right: Box::new(right.cast_to(&super_type, schema)?),
                    })
                }
            }
            Expr::IsNull(e) => Ok(Expr::IsNull(Box::new(self.rewrite_expr(e, schema)?))),
            Expr::IsNotNull(e) => {
                Ok(Expr::IsNotNull(Box::new(self.rewrite_expr(e, schema)?)))
            }
            Expr::ScalarFunction {
                name,
                args,
                return_type,
            } => {
                // cast the inputs of scalar functions to the appropriate type where possible
                match self.scalar_functions.get(name) {
                    Some(func_meta) => {
                        let mut func_args = Vec::with_capacity(args.len());
                        for i in 0..args.len() {
                            let field = &func_meta.args[i];
                            let expr = self.rewrite_expr(&args[i], schema)?;
                            let actual_type = expr.get_type(schema)?;
                            let required_type = field.data_type();
                            if &actual_type == required_type {
                                func_args.push(expr)
                            } else {
                                let super_type =
                                    utils::get_supertype(&actual_type, required_type)?;
                                func_args.push(expr.cast_to(&super_type, schema)?);
                            }
                        }

                        Ok(Expr::ScalarFunction {
                            name: name.clone(),
                            args: func_args,
                            return_type: return_type.clone(),
                        })
                    }
                    _ => Err(ExecutionError::General(format!(
                        "Invalid scalar function {}",
                        name
                    ))),
                }
            }
            Expr::AggregateFunction {
                name,
                args,
                return_type,
            } => Ok(Expr::AggregateFunction {
                name: name.clone(),
                args: args
                    .iter()
                    .map(|a| self.rewrite_expr(a, schema))
                    .collect::<Result<Vec<_>>>()?,
                return_type: return_type.clone(),
            }),
            Expr::Cast { .. } => Ok(expr.clone()),
            Expr::Column(_) => Ok(expr.clone()),
            Expr::Alias(expr, alias) => Ok(Expr::Alias(
                Box::new(self.rewrite_expr(expr, schema)?),
                alias.to_owned(),
            )),
            Expr::Literal(_) => Ok(expr.clone()),
            Expr::UnresolvedColumn(_) => Ok(expr.clone()),
            Expr::Not(_) => Ok(expr.clone()),
            Expr::Sort { .. } => Ok(expr.clone()),
            Expr::Wildcard { .. } => Err(ExecutionError::General(
                "Wildcard expressions are not valid in a logical query plan".to_owned(),
            )),
        }
    }
}

impl<'a> OptimizerRule for TypeCoercionRule<'a> {
    fn optimize(&mut self, plan: &LogicalPlan) -> Result<LogicalPlan> {
        match plan {
            LogicalPlan::Projection { expr, input, .. } => {
                LogicalPlanBuilder::from(&self.optimize(input)?)
                    .project(self.rewrite_expr_list(expr, input.schema())?)?
                    .build()
            }
            LogicalPlan::Selection { expr, input, .. } => {
                LogicalPlanBuilder::from(&self.optimize(input)?)
                    .filter(self.rewrite_expr(expr, input.schema())?)?
                    .build()
            }
            LogicalPlan::Aggregate {
                input,
                group_expr,
                aggr_expr,
                ..
            } => LogicalPlanBuilder::from(&self.optimize(input)?)
                .aggregate(
                    self.rewrite_expr_list(group_expr, input.schema())?,
                    self.rewrite_expr_list(aggr_expr, input.schema())?,
                )?
                .build(),
            LogicalPlan::TableScan { .. } => Ok(plan.clone()),
            LogicalPlan::EmptyRelation { .. } => Ok(plan.clone()),
            LogicalPlan::Limit { .. } => Ok(plan.clone()),
            LogicalPlan::CreateExternalTable { .. } => Ok(plan.clone()),
            other => Err(ExecutionError::NotImplemented(format!(
                "Type coercion optimizer rule does not support relation: {:?}",
                other
            ))),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::execution::context::ExecutionContext;
    use crate::logicalplan::Expr::*;
    use crate::logicalplan::Operator;
    use arrow::datatypes::{DataType, Field, Schema};

    #[test]
    fn test_add_i32_i64() {
        binary_cast_test(
            DataType::Int32,
            DataType::Int64,
            "CAST(#0 AS Int64) Plus #1",
        );
        binary_cast_test(
            DataType::Int64,
            DataType::Int32,
            "#0 Plus CAST(#1 AS Int64)",
        );
    }

    #[test]
    fn test_add_f32_f64() {
        binary_cast_test(
            DataType::Float32,
            DataType::Float64,
            "CAST(#0 AS Float64) Plus #1",
        );
        binary_cast_test(
            DataType::Float64,
            DataType::Float32,
            "#0 Plus CAST(#1 AS Float64)",
        );
    }

    #[test]
    fn test_add_i32_f32() {
        binary_cast_test(
            DataType::Int32,
            DataType::Float32,
            "CAST(#0 AS Float32) Plus #1",
        );
        binary_cast_test(
            DataType::Float32,
            DataType::Int32,
            "#0 Plus CAST(#1 AS Float32)",
        );
    }

    #[test]
    fn test_add_u32_i64() {
        binary_cast_test(
            DataType::UInt32,
            DataType::Int64,
            "CAST(#0 AS Int64) Plus #1",
        );
        binary_cast_test(
            DataType::Int64,
            DataType::UInt32,
            "#0 Plus CAST(#1 AS Int64)",
        );
    }

    fn binary_cast_test(left_type: DataType, right_type: DataType, expected: &str) {
        let schema = Schema::new(vec![
            Field::new("c0", left_type, true),
            Field::new("c1", right_type, true),
        ]);

        let expr = Expr::BinaryExpr {
            left: Box::new(Column(0)),
            op: Operator::Plus,
            right: Box::new(Column(1)),
        };

        let ctx = ExecutionContext::new();
        let rule = TypeCoercionRule::new(ctx.scalar_functions());

        let expr2 = rule.rewrite_expr(&expr, &schema).unwrap();

        assert_eq!(expected, format!("{:?}", expr2));
    }
}
<commit_msg>ARROW-8869: [Rust] [DataFusion] Add support for new scan nodes to type coercion rule<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

//! The type_coercion optimizer rule ensures that all operators are operating on
//! compatible types by adding explicit cast operations to expressions. For example,
//! the operation `c_float + c_int` would be rewritten as `c_float + CAST(c_int AS
//! float)`. This keeps the runtime query execution code much simpler.

use std::collections::HashMap;

use arrow::datatypes::Schema;

use crate::error::{ExecutionError, Result};
use crate::execution::physical_plan::udf::ScalarFunction;
use crate::logicalplan::LogicalPlan;
use crate::logicalplan::{Expr, LogicalPlanBuilder};
use crate::optimizer::optimizer::OptimizerRule;
use crate::optimizer::utils;

/// Implementation of type coercion optimizer rule
pub struct TypeCoercionRule<'a> {
    scalar_functions: &'a HashMap<String, Box<ScalarFunction>>,
}

impl<'a> TypeCoercionRule<'a> {
    /// Create a new type coercion optimizer rule using meta-data about registered
    /// scalar functions
    pub fn new(scalar_functions: &'a HashMap<String, Box<ScalarFunction>>) -> Self {
        Self { scalar_functions }
    }

    /// Rewrite an expression list to include explicit CAST operations when required
    fn rewrite_expr_list(&self, expr: &[Expr], schema: &Schema) -> Result<Vec<Expr>> {
        Ok(expr
            .iter()
            .map(|e| self.rewrite_expr(e, schema))
            .collect::<Result<Vec<_>>>()?)
    }

    /// Rewrite an expression to include explicit CAST operations when required
    fn rewrite_expr(&self, expr: &Expr, schema: &Schema) -> Result<Expr> {
        match expr {
            Expr::BinaryExpr { left, op, right } => {
                let left = self.rewrite_expr(left, schema)?;
                let right = self.rewrite_expr(right, schema)?;
                let left_type = left.get_type(schema)?;
                let right_type = right.get_type(schema)?;
                if left_type == right_type {
                    Ok(Expr::BinaryExpr {
                        left: Box::new(left),
                        op: op.clone(),
                        right: Box::new(right),
                    })
                } else {
                    let super_type = utils::get_supertype(&left_type, &right_type)?;
                    Ok(Expr::BinaryExpr {
                        left: Box::new(left.cast_to(&super_type, schema)?),
                        op: op.clone(),
                        right: Box::new(right.cast_to(&super_type, schema)?),
                    })
                }
            }
            Expr::IsNull(e) => Ok(Expr::IsNull(Box::new(self.rewrite_expr(e, schema)?))),
            Expr::IsNotNull(e) => {
                Ok(Expr::IsNotNull(Box::new(self.rewrite_expr(e, schema)?)))
            }
            Expr::ScalarFunction {
                name,
                args,
                return_type,
            } => {
                // cast the inputs of scalar functions to the appropriate type where possible
                match self.scalar_functions.get(name) {
                    Some(func_meta) => {
                        let mut func_args = Vec::with_capacity(args.len());
                        for i in 0..args.len() {
                            let field = &func_meta.args[i];
                            let expr = self.rewrite_expr(&args[i], schema)?;
                            let actual_type = expr.get_type(schema)?;
                            let required_type = field.data_type();
                            if &actual_type == required_type {
                                func_args.push(expr)
                            } else {
                                let super_type =
                                    utils::get_supertype(&actual_type, required_type)?;
                                func_args.push(expr.cast_to(&super_type, schema)?);
                            }
                        }

                        Ok(Expr::ScalarFunction {
                            name: name.clone(),
                            args: func_args,
                            return_type: return_type.clone(),
                        })
                    }
                    _ => Err(ExecutionError::General(format!(
                        "Invalid scalar function {}",
                        name
                    ))),
                }
            }
            Expr::AggregateFunction {
                name,
                args,
                return_type,
            } => Ok(Expr::AggregateFunction {
                name: name.clone(),
                args: args
                    .iter()
                    .map(|a| self.rewrite_expr(a, schema))
                    .collect::<Result<Vec<_>>>()?,
                return_type: return_type.clone(),
            }),
            Expr::Cast { .. } => Ok(expr.clone()),
            Expr::Column(_) => Ok(expr.clone()),
            Expr::Alias(expr, alias) => Ok(Expr::Alias(
                Box::new(self.rewrite_expr(expr, schema)?),
                alias.to_owned(),
            )),
            Expr::Literal(_) => Ok(expr.clone()),
            Expr::UnresolvedColumn(_) => Ok(expr.clone()),
            Expr::Not(_) => Ok(expr.clone()),
            Expr::Sort { .. } => Ok(expr.clone()),
            Expr::Wildcard { .. } => Err(ExecutionError::General(
                "Wildcard expressions are not valid in a logical query plan".to_owned(),
            )),
        }
    }
}

impl<'a> OptimizerRule for TypeCoercionRule<'a> {
    fn optimize(&mut self, plan: &LogicalPlan) -> Result<LogicalPlan> {
        match plan {
            LogicalPlan::Projection { expr, input, .. } => {
                LogicalPlanBuilder::from(&self.optimize(input)?)
                    .project(self.rewrite_expr_list(expr, input.schema())?)?
                    .build()
            }
            LogicalPlan::Selection { expr, input, .. } => {
                LogicalPlanBuilder::from(&self.optimize(input)?)
                    .filter(self.rewrite_expr(expr, input.schema())?)?
                    .build()
            }
            LogicalPlan::Aggregate {
                input,
                group_expr,
                aggr_expr,
                ..
            } => LogicalPlanBuilder::from(&self.optimize(input)?)
                .aggregate(
                    self.rewrite_expr_list(group_expr, input.schema())?,
                    self.rewrite_expr_list(aggr_expr, input.schema())?,
                )?
                .build(),
            LogicalPlan::TableScan { .. } => Ok(plan.clone()),
            LogicalPlan::InMemoryScan { .. } => Ok(plan.clone()),
            LogicalPlan::ParquetScan { .. } => Ok(plan.clone()),
            LogicalPlan::CsvScan { .. } => Ok(plan.clone()),
            LogicalPlan::EmptyRelation { .. } => Ok(plan.clone()),
            LogicalPlan::Limit { .. } => Ok(plan.clone()),
            LogicalPlan::CreateExternalTable { .. } => Ok(plan.clone()),
            other => Err(ExecutionError::NotImplemented(format!(
                "Type coercion optimizer rule does not support relation: {:?}",
                other
            ))),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::execution::context::ExecutionContext;
    use crate::execution::physical_plan::csv::CsvReadOptions;
    use crate::logicalplan::Expr::*;
    use crate::logicalplan::{col, Operator};
    use crate::test::arrow_testdata_path;
    use arrow::datatypes::{DataType, Field, Schema};

    #[test]
    fn test_with_csv_plan() -> Result<()> {
        let testdata = arrow_testdata_path();
        let path = format!("{}/csv/aggregate_test_100.csv", testdata);

        let options = CsvReadOptions::new().schema_infer_max_records(100);
        let plan = LogicalPlanBuilder::scan_csv(&path, options, None)?
            .filter(col("c7").lt(&col("c12")))?
            .build()?;

        let scalar_functions = HashMap::new();
        let mut rule = TypeCoercionRule::new(&scalar_functions);
        let plan = rule.optimize(&plan)?;

        assert!(
            format!("{:?}", plan).starts_with("Selection: CAST(#c7 AS Float64) Lt #c12")
        );

        Ok(())
    }

    #[test]
    fn test_add_i32_i64() {
        binary_cast_test(
            DataType::Int32,
            DataType::Int64,
            "CAST(#0 AS Int64) Plus #1",
        );
        binary_cast_test(
            DataType::Int64,
            DataType::Int32,
            "#0 Plus CAST(#1 AS Int64)",
        );
    }

    #[test]
    fn test_add_f32_f64() {
        binary_cast_test(
            DataType::Float32,
            DataType::Float64,
            "CAST(#0 AS Float64) Plus #1",
        );
        binary_cast_test(
            DataType::Float64,
            DataType::Float32,
            "#0 Plus CAST(#1 AS Float64)",
        );
    }

    #[test]
    fn test_add_i32_f32() {
        binary_cast_test(
            DataType::Int32,
            DataType::Float32,
            "CAST(#0 AS Float32) Plus #1",
        );
        binary_cast_test(
            DataType::Float32,
            DataType::Int32,
            "#0 Plus CAST(#1 AS Float32)",
        );
    }

    #[test]
    fn test_add_u32_i64() {
        binary_cast_test(
            DataType::UInt32,
            DataType::Int64,
            "CAST(#0 AS Int64) Plus #1",
        );
        binary_cast_test(
            DataType::Int64,
            DataType::UInt32,
            "#0 Plus CAST(#1 AS Int64)",
        );
    }

    fn binary_cast_test(left_type: DataType, right_type: DataType, expected: &str) {
        let schema = Schema::new(vec![
            Field::new("c0", left_type, true),
            Field::new("c1", right_type, true),
        ]);

        let expr = Expr::BinaryExpr {
            left: Box::new(Column(0)),
            op: Operator::Plus,
            right: Box::new(Column(1)),
        };

        let ctx = ExecutionContext::new();
        let rule = TypeCoercionRule::new(ctx.scalar_functions());

        let expr2 = rule.rewrite_expr(&expr, &schema).unwrap();

        assert_eq!(expected, format!("{:?}", expr2));
    }
}
<|endoftext|>