<commit_before>//! Procedural macro for the `#[async]` attribute.
//!
//! This crate is an implementation of the `#[async]` attribute as a procedural
//! macro. This is nightly-only for now as it's using the unstable features of
//! procedural macros. Furthermore it's generating code that's using a new
//! keyword, `yield`, and a new construct, generators, both of which are also
//! unstable.
//!
//! Currently this crate depends on `syn` and `quote` to do all the heavy
//! lifting, this is just a very small shim around creating a closure/future out
//! of a generator.

#![feature(proc_macro)]
#![recursion_limit = "128"]

extern crate proc_macro;
extern crate proc_macro2;
#[macro_use]
extern crate quote;
extern crate syn;

use proc_macro::TokenStream;
use syn::*;
use syn::fold::Folder;

#[proc_macro_attribute]
pub fn async(attribute: TokenStream, function: TokenStream) -> TokenStream {
    // Handle arguments to the #[async] attribute, if any
    let attribute = attribute.to_string();
    let boxed = if attribute == "( boxed )" {
        true
    } else if attribute == "" {
        false
    } else {
        panic!("the #[async] attribute currently only takes `boxed` as an arg");
    };

    // Parse our item, expecting a function. This function may be an actual
    // top-level function or it could be a method (typically dictated by the
    // arguments). We then extract everything we'd like to use.
    let function = proc_macro2::TokenStream::from(function);
    let Item { ident, vis, attrs, node } = function.into();
    let ItemFn {
        unsafety,
        constness,
        abi,
        block,
        decl,
        ..
    } = match node {
        ItemKind::Fn(item) => item,
        _ => panic!("#[async] can only be applied to functions"),
    };
    let FnDecl { inputs, output, variadic, generics, .. } = { *decl };
    let where_clause = &generics.where_clause;
    assert!(!variadic, "variadic functions cannot be async");
    let ref inputs = inputs;
    let output = match output {
        FunctionRetTy::Ty(t, _) => t,
        FunctionRetTy::Default => {
            TyTup {
                tys: Default::default(),
                lone_comma: Default::default(),
                paren_token: Default::default(),
            }.into()
        }
    };

    // We've got to get a bit creative with our handling of arguments. For a
    // number of reasons we translate this:
    //
    //      fn foo(ref a: u32) -> Result<u32, u32> {
    //          // ...
    //      }
    //
    // into roughly:
    //
    //      fn foo(__arg_0: u32) -> impl Future<...> {
    //          gen(move || {
    //              let ref a = __arg0;
    //
    //              // ...
    //          })
    //      }
    //
    // The intention here is to ensure that all local function variables get
    // moved into the generator we're creating, and they're also all then bound
    // appropriately according to their patterns and whatnot.
    //
    // We notably skip everything related to `self` which typically doesn't have
    // many patterns with it and just gets captured naturally.
    let mut inputs_no_patterns = Vec::new();
    let mut patterns = Vec::new();
    let mut temp_bindings = Vec::new();
    for (i, input) in inputs.iter().enumerate() {
        let input = *input.item();

        // `self: Box<Self>` will get captured naturally
        if let FnArg::Captured(ref arg) = *input {
            if let Pat::Ident(PatIdent { ref ident, ..}) = arg.pat {
                if ident == "self" {
                    inputs_no_patterns.push(input.clone());
                    continue
                }
            }
        }

        match *input {
            // `a: B`
            FnArg::Captured(ArgCaptured { ref pat, ref ty, .. }) => {
                patterns.push(pat);
                let ident = Ident::from(format!("__arg_{}", i));
                temp_bindings.push(ident.clone());
                let pat = PatIdent {
                    mode: BindingMode::ByValue(Mutability::Immutable),
                    ident: ident,
                    at_token: None,
                    subpat: None,
                };
                inputs_no_patterns.push(ArgCaptured {
                    pat: pat.into(),
                    ty: ty.clone(),
                    colon_token: Default::default(),
                }.into());
            }

            // Other `self`-related arguments get captured naturally
            _ => {
                inputs_no_patterns.push(input.clone());
            }
        }
    }

    // This is the point where we Handle
    //
    //      #[async]
    //      for x in y {
    //      }
    //
    // Basically just take all those expression and expand them.
    let block = ExpandAsyncFor.fold_block(*block);

    // TODO: can we lift the restriction that `futures` must be at the root of
    //       the crate?
    let return_ty = if boxed {
        quote! {
            Box<::futures::Future<
                Item = <#output as ::futures::__rt::FutureType>::Item,
                Error = <#output as ::futures::__rt::FutureType>::Error,
            >>
        }
    } else {
        // Dunno why this is buggy, hits weird typecheck errors in
        // `examples/main.rs`
        // -> impl ::futures::Future<
        //         Item = <#output as ::futures::__rt::FutureType>::Item,
        //         Error = <#output as ::futures::__rt::FutureType>::Error,
        //    >

        quote! { impl ::futures::__rt::MyFuture<#output> }
    };

    let maybe_boxed = if boxed {
        quote! { Box::new }
    } else {
        quote! { }
    };

    let output = quote! {
        #(#attrs)*
        #vis #unsafety #abi #constness
        fn #ident #generics(#(#inputs_no_patterns),*) -> #return_ty
            #where_clause
        {
            #maybe_boxed (::futures::__rt::gen((move || {
                let __e = {
                    #( let #patterns = #temp_bindings; )*
                    #block
                };

                // Ensure that this closure is a generator, even if it doesn't
                // have any `yield` statements.
                #[allow(unreachable_code)]
                {
                    return __e;
                    loop { yield }
                }
            })()))
        }
    };

    // println!("{}", output);
    let output: proc_macro2::TokenStream = output.into();
    output.into()
}

struct ExpandAsyncFor;

impl Folder for ExpandAsyncFor {
    fn fold_expr(&mut self, expr: Expr) -> Expr {
        if expr.attrs.len() != 1 {
            return fold::noop_fold_expr(self, expr)
        }
        // TODO: more validation here
        if expr.attrs[0].path.segments.get(0).item().ident != "async" {
            return expr
        }
        let all = match expr.node {
            ExprKind::ForLoop(item) => item,
            _ => panic!("only for expressions can have #[async]"),
        };
        let ExprForLoop { pat, expr, body, label, colon_token, .. } = all;

        // Basically just expand to a `poll` loop
        let tokens = quote! {{
            let mut __stream = #expr;
            #label
            #colon_token
            loop {
                let #pat = {
                    extern crate futures_await;
                    let r = futures_await::Stream::poll(&mut __stream)?;
                    match r {
                        futures_await::Async::Ready(e) => {
                            match e {
                                futures_await::__rt::Some(e) => e,
                                futures_await::__rt::None => break,
                            }
                        }
                        futures_await::Async::NotReady => {
                            yield;
                            continue
                        }
                    }
                };

                #body
            }
        }};
        proc_macro2::TokenStream::from(tokens).into()
    }

    // Don't recurse into items
    fn fold_item(&mut self, item: Item) -> Item {
        item
    }
}
<commit_msg>Update to the tip of `gen` branch<commit_after>//! Procedural macro for the `#[async]` attribute.
//!
//! This crate is an implementation of the `#[async]` attribute as a procedural
//! macro. This is nightly-only for now as it's using the unstable features of
//! procedural macros. Furthermore it's generating code that's using a new
//! keyword, `yield`, and a new construct, generators, both of which are also
//! unstable.
//!
//! Currently this crate depends on `syn` and `quote` to do all the heavy
//! lifting, this is just a very small shim around creating a closure/future out
//! of a generator.

#![feature(proc_macro)]
#![recursion_limit = "128"]

extern crate proc_macro;
extern crate proc_macro2;
#[macro_use]
extern crate quote;
extern crate syn;

use proc_macro::TokenStream;
use syn::*;
use syn::fold::Folder;

#[proc_macro_attribute]
pub fn async(attribute: TokenStream, function: TokenStream) -> TokenStream {
    // Handle arguments to the #[async] attribute, if any
    let attribute = attribute.to_string();
    let boxed = if attribute == "( boxed )" {
        true
    } else if attribute == "" {
        false
    } else {
        panic!("the #[async] attribute currently only takes `boxed` as an arg");
    };

    // Parse our item, expecting a function. This function may be an actual
    // top-level function or it could be a method (typically dictated by the
    // arguments). We then extract everything we'd like to use.
    let function = proc_macro2::TokenStream::from(function);
    let Item { ident, vis, attrs, node } = function.into();
    let ItemFn {
        unsafety,
        constness,
        abi,
        block,
        decl,
        ..
    } = match node {
        ItemKind::Fn(item) => item,
        _ => panic!("#[async] can only be applied to functions"),
    };
    let FnDecl { inputs, output, variadic, generics, .. } = { *decl };
    let where_clause = &generics.where_clause;
    assert!(!variadic, "variadic functions cannot be async");
    let ref inputs = inputs;
    let output = match output {
        FunctionRetTy::Ty(t, _) => t,
        FunctionRetTy::Default => {
            TyTup {
                tys: Default::default(),
                lone_comma: Default::default(),
                paren_token: Default::default(),
            }.into()
        }
    };

    // We've got to get a bit creative with our handling of arguments. For a
    // number of reasons we translate this:
    //
    //      fn foo(ref a: u32) -> Result<u32, u32> {
    //          // ...
    //      }
    //
    // into roughly:
    //
    //      fn foo(__arg_0: u32) -> impl Future<...> {
    //          gen(move || {
    //              let ref a = __arg0;
    //
    //              // ...
    //          })
    //      }
    //
    // The intention here is to ensure that all local function variables get
    // moved into the generator we're creating, and they're also all then bound
    // appropriately according to their patterns and whatnot.
    //
    // We notably skip everything related to `self` which typically doesn't have
    // many patterns with it and just gets captured naturally.
    let mut inputs_no_patterns = Vec::new();
    let mut patterns = Vec::new();
    let mut temp_bindings = Vec::new();
    for (i, input) in inputs.iter().enumerate() {
        let input = *input.item();

        // `self: Box<Self>` will get captured naturally
        if let FnArg::Captured(ref arg) = *input {
            if let Pat::Ident(PatIdent { ref ident, ..}) = arg.pat {
                if ident == "self" {
                    inputs_no_patterns.push(input.clone());
                    continue
                }
            }
        }

        match *input {
            // `a: B`
            FnArg::Captured(ArgCaptured { ref pat, ref ty, .. }) => {
                patterns.push(pat);
                let ident = Ident::from(format!("__arg_{}", i));
                temp_bindings.push(ident.clone());
                let pat = PatIdent {
                    mode: BindingMode::ByValue(Mutability::Immutable),
                    ident: ident,
                    at_token: None,
                    subpat: None,
                };
                inputs_no_patterns.push(ArgCaptured {
                    pat: pat.into(),
                    ty: ty.clone(),
                    colon_token: Default::default(),
                }.into());
            }

            // Other `self`-related arguments get captured naturally
            _ => {
                inputs_no_patterns.push(input.clone());
            }
        }
    }

    // This is the point where we Handle
    //
    //      #[async]
    //      for x in y {
    //      }
    //
    // Basically just take all those expression and expand them.
    let block = ExpandAsyncFor.fold_block(*block);

    // TODO: can we lift the restriction that `futures` must be at the root of
    //       the crate?
    let return_ty = if boxed {
        quote! {
            Box<::futures::Future<
                Item = <#output as ::futures::__rt::FutureType>::Item,
                Error = <#output as ::futures::__rt::FutureType>::Error,
            >>
        }
    } else {
        // Dunno why this is buggy, hits weird typecheck errors in
        // `examples/main.rs`
        // -> impl ::futures::Future<
        //         Item = <#output as ::futures::__rt::FutureType>::Item,
        //         Error = <#output as ::futures::__rt::FutureType>::Error,
        //    >

        quote! { impl ::futures::__rt::MyFuture<#output> }
    };

    let maybe_boxed = if boxed {
        quote! { Box::new }
    } else {
        quote! { }
    };

    let output = quote! {
        #(#attrs)*
        #vis #unsafety #abi #constness
        fn #ident #generics(#(#inputs_no_patterns),*) -> #return_ty
            #where_clause
        {
            #maybe_boxed (::futures::__rt::gen(move || {
                let __e = {
                    #( let #patterns = #temp_bindings; )*
                    #block
                };

                // Ensure that this closure is a generator, even if it doesn't
                // have any `yield` statements.
                #[allow(unreachable_code)]
                {
                    return __e;
                    loop { yield }
                }
            }))
        }
    };

    // println!("{}", output);
    let output: proc_macro2::TokenStream = output.into();
    output.into()
}

struct ExpandAsyncFor;

impl Folder for ExpandAsyncFor {
    fn fold_expr(&mut self, expr: Expr) -> Expr {
        if expr.attrs.len() != 1 {
            return fold::noop_fold_expr(self, expr)
        }
        // TODO: more validation here
        if expr.attrs[0].path.segments.get(0).item().ident != "async" {
            return expr
        }
        let all = match expr.node {
            ExprKind::ForLoop(item) => item,
            _ => panic!("only for expressions can have #[async]"),
        };
        let ExprForLoop { pat, expr, body, label, colon_token, .. } = all;

        // Basically just expand to a `poll` loop
        let tokens = quote! {{
            let mut __stream = #expr;
            #label
            #colon_token
            loop {
                let #pat = {
                    extern crate futures_await;
                    let r = futures_await::Stream::poll(&mut __stream)?;
                    match r {
                        futures_await::Async::Ready(e) => {
                            match e {
                                futures_await::__rt::Some(e) => e,
                                futures_await::__rt::None => break,
                            }
                        }
                        futures_await::Async::NotReady => {
                            yield;
                            continue
                        }
                    }
                };

                #body
            }
        }};
        proc_macro2::TokenStream::from(tokens).into()
    }

    // Don't recurse into items
    fn fold_item(&mut self, item: Item) -> Item {
        item
    }
}
<|endoftext|>


<commit_before>use jpeg;
use png;
use std::cmp;
use std::fs::File;
use std::path::Path;

use super::common;

#[test]
fn reftest() {
    let files = common::test_files(&Path::new("tests").join("reftest").join("images"));

    for path in &files {
        reftest_file(path);
    }
}

#[test]
fn reftest_scaled() {
    let base = &Path::new("tests").join("reftest").join("images");
    reftest_scaled_file(&base.join("rgb.jpg"), 500, 333, &base.join("rgb.png"));
    reftest_scaled_file(&base.join("rgb.jpg"), 250, 167, &base.join("rgb_250x167.png"));
    reftest_scaled_file(&base.join("rgb.jpg"), 125, 84,  &base.join("rgb_125x84.png"));
    reftest_scaled_file(&base.join("rgb.jpg"), 63,  42,  &base.join("rgb_63x42.png"));
}

fn reftest_file(path: &Path) {
    let file = File::open(path).unwrap();
    let decoder = jpeg::Decoder::new(file);
    reftest_decoder(decoder, path, &path.with_extension("png"));
}

fn reftest_scaled_file(path: &Path, width: u16, height: u16, ref_path: &Path) {
    let file = File::open(path).unwrap();
    let mut decoder = jpeg::Decoder::new(file);
    decoder.read_info().unwrap();
    decoder.scale(width, height).unwrap();
    reftest_decoder(decoder, path, &ref_path);
}

fn reftest_decoder<T: std::io::Read>(mut decoder: jpeg::Decoder<T>, path: &Path, ref_path: &Path) {
    let mut data = decoder.decode().expect(&format!("failed to decode file: {:?}", path));
    let info = decoder.info().unwrap();
    let mut pixel_format = info.pixel_format;

    if pixel_format == jpeg::PixelFormat::CMYK32 {
        data = cmyk_to_rgb(&data);
        pixel_format = jpeg::PixelFormat::RGB24;
    }

    let ref_file = File::open(ref_path).unwrap();
    let mut decoder = png::Decoder::new(ref_file);
    decoder.set_transformations(png::Transformations::EXPAND);
    let (ref_info, mut ref_reader) = decoder.read_info().expect("png failed to read info");
    println!("ref_info {:?}", ref_info);

    assert_eq!(ref_info.width, info.width as u32);
    assert_eq!(ref_info.height, info.height as u32);

    let mut ref_data = vec![0; ref_info.buffer_size()];
    ref_reader.next_frame(&mut ref_data).expect("png decode failed");
    let mut ref_pixel_format = ref_info.color_type;

    if ref_pixel_format == png::ColorType::RGBA {
        let refdatai : Vec<isize> = ref_data.iter().map(|x| *x as isize).collect();
        ref_data = rgba_to_rgb(&refdatai).iter().map(|x| *x as u8).collect();
        ref_pixel_format = png::ColorType::RGB;
    }

    match pixel_format {
        jpeg::PixelFormat::L8 => {
            assert_eq!(ref_pixel_format, png::ColorType::Grayscale);
            assert_eq!(ref_info.bit_depth, png::BitDepth::Eight);
        },
        jpeg::PixelFormat::L16 => {
            assert_eq!(ref_pixel_format, png::ColorType::Grayscale);
            assert_eq!(ref_info.bit_depth, png::BitDepth::Sixteen);
        },
        jpeg::PixelFormat::RGB24 => {
            assert_eq!(ref_pixel_format, png::ColorType::RGB);
            assert_eq!(ref_info.bit_depth, png::BitDepth::Eight);
        },
        _ => panic!(),
    }

    assert_eq!(data.len(), ref_data.len());
    let mut max_diff = 0;
    let pixels: Vec<u8> = data.iter().zip(ref_data.iter()).map(|(&a, &b)| {
        let diff = (a as isize - b as isize).abs();
        max_diff = cmp::max(diff, max_diff);

        // FIXME: Only a diff of 1 should be allowed?
        if diff <= 2 {
            // White for correct
            0xFF
        } else {
            println!("data: {}, ref: {}", a, b);
            // "1100" in the RGBA channel with an error for an incorrect value
            // This results in some number of C0 and FFs, which is much more
            // readable (and distinguishable) than the previous difference-wise
            // scaling but does not require reconstructing the actual RGBA pixel.
            0xC0
        }
    }).collect();

    if pixels.iter().any(|&a| a < 255) {
        let output_path = path.with_file_name(format!("{}-diff.png", path.file_stem().unwrap().to_str().unwrap()));
        let output = File::create(&output_path).unwrap();
        let mut encoder = png::Encoder::new(output, info.width as u32, info.height as u32);
        encoder.set_depth(png::BitDepth::Eight);
        encoder.set_color(ref_pixel_format);
        encoder.write_header().expect("png failed to write header").write_image_data(&pixels).expect("png failed to write data");

        panic!("decoding difference: {:?}, maximum difference was {}", output_path, max_diff);
    }
}

fn rgba_to_rgb(input: &[isize]) -> Vec<isize> {
    let size = input.len() - input.len() / 4;
    let mut output = Vec::with_capacity(size);

    for pixel in input.chunks(4) {
        assert_eq!(pixel[3], 255);

        output.push(pixel[0]);
        output.push(pixel[1]);
        output.push(pixel[2]);
    }

    output
}

fn cmyk_to_rgb(input: &[isize]) -> Vec<isize> {
    let size = input.len() - input.len() / 4;
    let mut output = Vec::with_capacity(size);

    for pixel in input.chunks(4) {
        let c = pixel[0] as f32 / 255.0;
        let m = pixel[1] as f32 / 255.0;
        let y = pixel[2] as f32 / 255.0;
        let k = pixel[3] as f32 / 255.0;

        // CMYK -> CMY
        let c = c * (1.0 - k) + k;
        let m = m * (1.0 - k) + k;
        let y = y * (1.0 - k) + k;

        // CMY -> RGB
        let r = (1.0 - c) * 255.0;
        let g = (1.0 - m) * 255.0;
        let b = (1.0 - y) * 255.0;

        output.push(r as isize);
        output.push(g as isize);
        output.push(b as isize);
    }

    output
}
<commit_msg>tests pass<commit_after>use jpeg;
use png;
use std::cmp;
use std::fs::File;
use std::path::Path;

use super::common;

#[test]
fn reftest() {
    let files = common::test_files(&Path::new("tests").join("reftest").join("images"));

    for path in &files {
        reftest_file(path);
    }
}

#[test]
fn reftest_scaled() {
    let base = &Path::new("tests").join("reftest").join("images");
    reftest_scaled_file(&base.join("rgb.jpg"), 500, 333, &base.join("rgb.png"));
    reftest_scaled_file(&base.join("rgb.jpg"), 250, 167, &base.join("rgb_250x167.png"));
    reftest_scaled_file(&base.join("rgb.jpg"), 125, 84,  &base.join("rgb_125x84.png"));
    reftest_scaled_file(&base.join("rgb.jpg"), 63,  42,  &base.join("rgb_63x42.png"));
}

fn reftest_file(path: &Path) {
    let file = File::open(path).unwrap();
    let decoder = jpeg::Decoder::new(file);
    reftest_decoder(decoder, path, &path.with_extension("png"));
}

fn reftest_scaled_file(path: &Path, width: u16, height: u16, ref_path: &Path) {
    let file = File::open(path).unwrap();
    let mut decoder = jpeg::Decoder::new(file);
    decoder.read_info().unwrap();
    decoder.scale(width, height).unwrap();
    reftest_decoder(decoder, path, &ref_path);
}

fn reftest_decoder<T: std::io::Read>(mut decoder: jpeg::Decoder<T>, path: &Path, ref_path: &Path) {
    let mut data = decoder.decode().expect(&format!("failed to decode file: {:?}", path));
    let info = decoder.info().unwrap();
    let mut pixel_format = info.pixel_format;

    if pixel_format == jpeg::PixelFormat::CMYK32 {
        data = cmyk_to_rgb(&data);
        pixel_format = jpeg::PixelFormat::RGB24;
    }

    let ref_file = File::open(ref_path).unwrap();
    let mut decoder = png::Decoder::new(ref_file);

    if ref_path.ends_with("lossless16bit.png") {
        // disable the default 8bit output of png v0.16.8 (changed in master branch of png)
        decoder.set_transformations(png::Transformations::EXPAND);
    }

    let (ref_info, mut ref_reader) = decoder.read_info().expect("png failed to read info");
    println!("ref_info {:?}", ref_info);

    assert_eq!(ref_info.width, info.width as u32);
    assert_eq!(ref_info.height, info.height as u32);

    let mut ref_data = vec![0; ref_info.buffer_size()];
    ref_reader.next_frame(&mut ref_data).expect("png decode failed");
    let mut ref_pixel_format = ref_info.color_type;

    let mut refdatai : Vec<isize> = ref_data.iter().map(|x| *x as isize).collect();

    if ref_pixel_format == png::ColorType::RGBA {
        refdatai = rgba_to_rgb(&refdatai);
        ref_pixel_format = png::ColorType::RGB;
    }

    match pixel_format {
        jpeg::PixelFormat::L8 => {
            assert_eq!(ref_pixel_format, png::ColorType::Grayscale);
            assert_eq!(ref_info.bit_depth, png::BitDepth::Eight);
        },
        jpeg::PixelFormat::L16 => {
            assert_eq!(ref_pixel_format, png::ColorType::Grayscale);
            assert_eq!(ref_info.bit_depth, png::BitDepth::Sixteen);
            refdatai = ref_data.chunks_exact(2).into_iter().map(|a| u16::from_be_bytes([a[0],a[1]])).map(|x| x as isize).collect();
        },
        jpeg::PixelFormat::RGB24 => {
            assert_eq!(ref_pixel_format, png::ColorType::RGB);
            assert_eq!(ref_info.bit_depth, png::BitDepth::Eight);
        },
        _ => panic!(),
    }

    assert_eq!(data.len(), refdatai.len());
    let mut max_diff = 0;
    let pixels: Vec<u8> = data.iter().zip(refdatai.iter()).map(|(&a, &b)| {
        let diff = (a as isize - b as isize).abs();
        max_diff = cmp::max(diff, max_diff);

        // FIXME: Only a diff of 1 should be allowed?
        if diff <= 2 {
            // White for correct
            0xFF
        } else {
            println!("data: {}, ref: {}", a, b);
            // "1100" in the RGBA channel with an error for an incorrect value
            // This results in some number of C0 and FFs, which is much more
            // readable (and distinguishable) than the previous difference-wise
            // scaling but does not require reconstructing the actual RGBA pixel.
            0xC0
        }
    }).collect();

    if pixels.iter().any(|&a| a < 255) {
        let output_path = path.with_file_name(format!("{}-diff.png", path.file_stem().unwrap().to_str().unwrap()));
        let output = File::create(&output_path).unwrap();
        let mut encoder = png::Encoder::new(output, info.width as u32, info.height as u32);
        encoder.set_depth(png::BitDepth::Eight);
        encoder.set_color(ref_pixel_format);
        encoder.write_header().expect("png failed to write header").write_image_data(&pixels).expect("png failed to write data");

        panic!("decoding difference: {:?}, maximum difference was {}", output_path, max_diff);
    }
}

fn rgba_to_rgb(input: &[isize]) -> Vec<isize> {
    let size = input.len() - input.len() / 4;
    let mut output = Vec::with_capacity(size);

    for pixel in input.chunks(4) {
        assert_eq!(pixel[3], 255);

        output.push(pixel[0]);
        output.push(pixel[1]);
        output.push(pixel[2]);
    }

    output
}

fn cmyk_to_rgb(input: &[isize]) -> Vec<isize> {
    let size = input.len() - input.len() / 4;
    let mut output = Vec::with_capacity(size);

    for pixel in input.chunks(4) {
        let c = pixel[0] as f32 / 255.0;
        let m = pixel[1] as f32 / 255.0;
        let y = pixel[2] as f32 / 255.0;
        let k = pixel[3] as f32 / 255.0;

        // CMYK -> CMY
        let c = c * (1.0 - k) + k;
        let m = m * (1.0 - k) + k;
        let y = y * (1.0 - k) + k;

        // CMY -> RGB
        let r = (1.0 - c) * 255.0;
        let g = (1.0 - m) * 255.0;
        let b = (1.0 - y) * 255.0;

        output.push(r as isize);
        output.push(g as isize);
        output.push(b as isize);
    }

    output
}
<|endoftext|>


<commit_before>// Copyright 2015-2016 Benjamin Fry <benjaminfry@me.com>
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.
use std;
use std::io;
use std::sync::Arc;
use std::time::Duration;

use futures::{Async, Future, Poll, Stream};

use tokio_core;
use tokio_core::reactor::Core;

use trust_dns::udp::UdpStream;
use trust_dns::tcp::TcpStream;

#[cfg(feature = "tls")]
use trust_dns_openssl::{tls_server, TlsStream};

#[cfg(feature = "tls")]
use trust_dns_openssl::tls_server::*;

use server::{Request, RequestHandler, RequestStream, ResponseHandle, TimeoutStream};

// TODO, would be nice to have a Slab for buffers here...

/// A Futures based implementation of a DNS server
pub struct ServerFuture<T: RequestHandler + 'static> {
    io_loop: Core,
    handler: Arc<T>,
}

impl<T: RequestHandler> ServerFuture<T> {
    /// Creates a new ServerFuture with the specified Handler.
    pub fn new(handler: T) -> io::Result<ServerFuture<T>> {
        Ok(ServerFuture {
            io_loop: Core::new()?,
            handler: Arc::new(handler),
        })
    }

    /// Register a UDP socket. Should be bound before calling this function.
    pub fn register_socket(&self, socket: std::net::UdpSocket) {
        debug!("registered udp: {:?}", socket);

        // create the new UdpStream
        let (buf_stream, stream_handle) = UdpStream::with_bound(socket, &self.io_loop.handle());
        let request_stream = RequestStream::new(buf_stream, stream_handle);
        let handler = self.handler.clone();

        // this spawns a ForEach future which handles all the requests into a Handler.
        self.io_loop.handle().spawn(
            // TODO dedup with below into generic func
            request_stream
                .for_each(move |(request, response_handle)| {
                    Self::handle_request(request, response_handle, handler.clone())
                })
                .map_err(|e| debug!("error in UDP request_stream handler: {}", e)),
        );
    }

    /// Register a TcpListener to the Server. This should already be bound to either an IPv6 or an
    ///  IPv4 address.
    ///
    /// To make the server more resilient to DOS issues, there is a timeout. Care should be taken
    ///  to not make this too low depending on use cases.
    ///
    /// # Arguments
    /// * `listener` - a bound TCP socket
    /// * `timeout` - timeout duration of incoming requests, any connection that does not send
    ///               requests within this time period will be closed. In the future it should be
    ///               possible to create long-lived queries, but these should be from trusted sources
    ///               only, this would require some type of whitelisting.
    pub fn register_listener(
        &self,
        listener: std::net::TcpListener,
        timeout: Duration,
    ) -> io::Result<()> {
        let handle = self.io_loop.handle();
        let handler = self.handler.clone();
        // TODO: this is an awkward interface with socketaddr...
        let addr = listener.local_addr()?;
        let listener = tokio_core::net::TcpListener::from_listener(listener, &addr, &handle)
            .expect("could not register listener");
        debug!("registered tcp: {:?}", listener);

        // for each incoming request...
        self.io_loop.handle().spawn(
            listener
                .incoming()
                .for_each(move |(tcp_stream, src_addr)| {
                    debug!("accepted request from: {}", src_addr);
                    // take the created stream...
                    let (buf_stream, stream_handle) = TcpStream::from_stream(tcp_stream, src_addr);
                    let timeout_stream = TimeoutStream::new(buf_stream, timeout, &handle)?;
                    let request_stream = RequestStream::new(timeout_stream, stream_handle);
                    let handler = handler.clone();

                    // and spawn to the io_loop
                    handle.spawn(
                        request_stream
                            .for_each(move |(request, response_handle)| {
                                Self::handle_request(request, response_handle, handler.clone())
                            })
                            .map_err(move |e| {
                                debug!(
                                    "error in TCP request_stream src: {:?} error: {}",
                                    src_addr,
                                    e
                                )
                            }),
                    );

                    Ok(())
                })
                .map_err(|e| debug!("error in inbound tcp_stream: {}", e)),
        );

        Ok(())
    }

    /// Register a TlsListener to the Server. The TlsListener should already be bound to either an
    /// IPv6 or an IPv4 address.
    ///
    /// To make the server more resilient to DOS issues, there is a timeout. Care should be taken
    ///  to not make this too low depending on use cases.
    ///
    /// # Arguments
    /// * `listener` - a bound TCP (needs to be on a different port from standard TCP connections) socket
    /// * `timeout` - timeout duration of incoming requests, any connection that does not send
    ///               requests within this time period will be closed. In the future it should be
    ///               possible to create long-lived queries, but these should be from trusted sources
    ///               only, this would require some type of whitelisting.
    /// * `pkcs12` - certificate used to announce to clients
    #[cfg(feature = "tls")]
    pub fn register_tls_listener(
        &self,
        listener: std::net::TcpListener,
        timeout: Duration,
        pkcs12: ParsedPkcs12,
    ) -> io::Result<()> {
        let handle = self.io_loop.handle();
        let handler = self.handler.clone();
        // TODO: this is an awkward interface with socketaddr...
        let addr = listener.local_addr().expect("listener is not bound?");
        let listener = tokio_core::net::TcpListener::from_listener(listener, &addr, &handle)
            .expect("could not register listener");
        debug!("registered tcp: {:?}", listener);

        let tls_acceptor = tls_server::new_acceptor(&pkcs12)?;

        // for each incoming request...
        self.io_loop.handle().spawn(
            listener
                .incoming()
                .for_each(move |(tcp_stream, src_addr)| {
                    debug!("accepted request from: {}", src_addr);
                    let timeout = timeout.clone();
                    let handle = handle.clone();
                    let handler = handler.clone();

                    // take the created stream...
                    tls_acceptor
                        .accept_async(tcp_stream)
                        .map_err(|e| {
                            io::Error::new(
                                io::ErrorKind::ConnectionRefused,
                                format!("tls error: {}", e),
                            )
                        })
                        .and_then(move |tls_stream| {
                            let (buf_stream, stream_handle) =
                                TlsStream::from_stream(tls_stream, src_addr.clone());
                            let timeout_stream = TimeoutStream::new(buf_stream, timeout, &handle)?;
                            let request_stream = RequestStream::new(timeout_stream, stream_handle);
                            let handler = handler.clone();

                            // and spawn to the io_loop
                            handle.spawn(
                                request_stream
                                    .for_each(move |(request, response_handle)| {
                                        Self::handle_request(
                                            request,
                                            response_handle,
                                            handler.clone(),
                                        )
                                    })
                                    .map_err(move |e| {
                                        debug!(
                                            "error in TCP request_stream src: {:?} error: {}",
                                            src_addr,
                                            e
                                        )
                                    }),
                            );

                            Ok(())
                        })
                    //.map_err(move |e| debug!("error TLS handshake: {:?} error: {:?}", src_addr, e))
                })
                .map_err(|e| debug!("error in inbound tcp_stream: {}", e)),
        );

        Ok(())
    }

    /// TODO: how to do threads? should we do a bunch of listener threads and then query threads?
    /// Ideally the processing would be n-threads for recieving, which hand off to m-threads for
    ///  request handling. It would generally be the case that n <= m.
    pub fn listen(&mut self) -> io::Result<()> {
        info!("Server starting up");
        self.io_loop.run(Forever)?;

        Err(io::Error::new(
            io::ErrorKind::Interrupted,
            "Server stopping due to interruption",
        ))
    }

    /// Returns a reference to the tokio core loop driving this Server instance
    pub fn tokio_core(&mut self) -> &mut Core {
        &mut self.io_loop
    }

    fn handle_request(
        request: Request,
        mut response_handle: ResponseHandle,
        handler: Arc<T>,
    ) -> io::Result<()> {
        info!(
            "request: {} type: {:?} op_code: {:?} dnssec: {} {}",
            request.message.id(),
            request.message.message_type(),
            request.message.op_code(),
            request
                .message
                .edns()
                .map_or(false, |edns| edns.dnssec_ok()),
            request
                .message
                .queries()
                .first()
                .map(|q| q.to_string())
                .unwrap_or_else(|| "empty_queries".to_string()),
        );
        let response = handler.handle_request(&request);

        info!(
            "request: {} response_code: {} answers: {} name_servers: {} additionals: {}",
            request.message.id(),
            response.response_code(),
            response.answers().len(),
            response.name_servers().len(),
            response.additionals().len(),
        );
        response_handle.send(response)
    }
}

struct Forever;

impl Future for Forever {
    type Item = ();
    type Error = io::Error;

    fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
        // run forever...
        Ok(Async::NotReady)
    }
}
<commit_msg>Remove some .clone calls<commit_after>// Copyright 2015-2016 Benjamin Fry <benjaminfry@me.com>
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.
use std;
use std::io;
use std::sync::Arc;
use std::time::Duration;

use futures::{Async, Future, Poll, Stream};

use tokio_core;
use tokio_core::reactor::Core;

use trust_dns::udp::UdpStream;
use trust_dns::tcp::TcpStream;

#[cfg(feature = "tls")]
use trust_dns_openssl::{tls_server, TlsStream};

#[cfg(feature = "tls")]
use trust_dns_openssl::tls_server::*;

use server::{Request, RequestHandler, RequestStream, ResponseHandle, TimeoutStream};

// TODO, would be nice to have a Slab for buffers here...

/// A Futures based implementation of a DNS server
pub struct ServerFuture<T: RequestHandler + 'static> {
    io_loop: Core,
    handler: Arc<T>,
}

impl<T: RequestHandler> ServerFuture<T> {
    /// Creates a new ServerFuture with the specified Handler.
    pub fn new(handler: T) -> io::Result<ServerFuture<T>> {
        Ok(ServerFuture {
            io_loop: Core::new()?,
            handler: Arc::new(handler),
        })
    }

    /// Register a UDP socket. Should be bound before calling this function.
    pub fn register_socket(&self, socket: std::net::UdpSocket) {
        debug!("registered udp: {:?}", socket);

        // create the new UdpStream
        let (buf_stream, stream_handle) = UdpStream::with_bound(socket, &self.io_loop.handle());
        let request_stream = RequestStream::new(buf_stream, stream_handle);
        let handler = self.handler.clone();

        // this spawns a ForEach future which handles all the requests into a Handler.
        self.io_loop.handle().spawn(
            // TODO dedup with below into generic func
            request_stream
                .for_each(move |(request, response_handle)| {
                    Self::handle_request(request, response_handle, handler.clone())
                })
                .map_err(|e| debug!("error in UDP request_stream handler: {}", e)),
        );
    }

    /// Register a TcpListener to the Server. This should already be bound to either an IPv6 or an
    ///  IPv4 address.
    ///
    /// To make the server more resilient to DOS issues, there is a timeout. Care should be taken
    ///  to not make this too low depending on use cases.
    ///
    /// # Arguments
    /// * `listener` - a bound TCP socket
    /// * `timeout` - timeout duration of incoming requests, any connection that does not send
    ///               requests within this time period will be closed. In the future it should be
    ///               possible to create long-lived queries, but these should be from trusted sources
    ///               only, this would require some type of whitelisting.
    pub fn register_listener(
        &self,
        listener: std::net::TcpListener,
        timeout: Duration,
    ) -> io::Result<()> {
        let handle = self.io_loop.handle();
        let handler = self.handler.clone();
        // TODO: this is an awkward interface with socketaddr...
        let addr = listener.local_addr()?;
        let listener = tokio_core::net::TcpListener::from_listener(listener, &addr, &handle)
            .expect("could not register listener");
        debug!("registered tcp: {:?}", listener);

        // for each incoming request...
        self.io_loop.handle().spawn(
            listener
                .incoming()
                .for_each(move |(tcp_stream, src_addr)| {
                    debug!("accepted request from: {}", src_addr);
                    // take the created stream...
                    let (buf_stream, stream_handle) = TcpStream::from_stream(tcp_stream, src_addr);
                    let timeout_stream = TimeoutStream::new(buf_stream, timeout, &handle)?;
                    let request_stream = RequestStream::new(timeout_stream, stream_handle);
                    let handler = handler.clone();

                    // and spawn to the io_loop
                    handle.spawn(
                        request_stream
                            .for_each(move |(request, response_handle)| {
                                Self::handle_request(request, response_handle, handler.clone())
                            })
                            .map_err(move |e| {
                                debug!(
                                    "error in TCP request_stream src: {:?} error: {}",
                                    src_addr,
                                    e
                                )
                            }),
                    );

                    Ok(())
                })
                .map_err(|e| debug!("error in inbound tcp_stream: {}", e)),
        );

        Ok(())
    }

    /// Register a TlsListener to the Server. The TlsListener should already be bound to either an
    /// IPv6 or an IPv4 address.
    ///
    /// To make the server more resilient to DOS issues, there is a timeout. Care should be taken
    ///  to not make this too low depending on use cases.
    ///
    /// # Arguments
    /// * `listener` - a bound TCP (needs to be on a different port from standard TCP connections) socket
    /// * `timeout` - timeout duration of incoming requests, any connection that does not send
    ///               requests within this time period will be closed. In the future it should be
    ///               possible to create long-lived queries, but these should be from trusted sources
    ///               only, this would require some type of whitelisting.
    /// * `pkcs12` - certificate used to announce to clients
    #[cfg(feature = "tls")]
    pub fn register_tls_listener(
        &self,
        listener: std::net::TcpListener,
        timeout: Duration,
        pkcs12: ParsedPkcs12,
    ) -> io::Result<()> {
        let handle = self.io_loop.handle();
        let handler = self.handler.clone();
        // TODO: this is an awkward interface with socketaddr...
        let addr = listener.local_addr().expect("listener is not bound?");
        let listener = tokio_core::net::TcpListener::from_listener(listener, &addr, &handle)
            .expect("could not register listener");
        debug!("registered tcp: {:?}", listener);

        let tls_acceptor = tls_server::new_acceptor(&pkcs12)?;

        // for each incoming request...
        self.io_loop.handle().spawn(
            listener
                .incoming()
                .for_each(move |(tcp_stream, src_addr)| {
                    debug!("accepted request from: {}", src_addr);
                    let handle = handle.clone();
                    let handler = handler.clone();

                    // take the created stream...
                    tls_acceptor
                        .accept_async(tcp_stream)
                        .map_err(|e| {
                            io::Error::new(
                                io::ErrorKind::ConnectionRefused,
                                format!("tls error: {}", e),
                            )
                        })
                        .and_then(move |tls_stream| {
                            let (buf_stream, stream_handle) =
                                TlsStream::from_stream(tls_stream, src_addr);
                            let timeout_stream = TimeoutStream::new(buf_stream, timeout, &handle)?;
                            let request_stream = RequestStream::new(timeout_stream, stream_handle);
                            let handler = handler.clone();

                            // and spawn to the io_loop
                            handle.spawn(
                                request_stream
                                    .for_each(move |(request, response_handle)| {
                                        Self::handle_request(
                                            request,
                                            response_handle,
                                            handler.clone(),
                                        )
                                    })
                                    .map_err(move |e| {
                                        debug!(
                                            "error in TCP request_stream src: {:?} error: {}",
                                            src_addr,
                                            e
                                        )
                                    }),
                            );

                            Ok(())
                        })
                    //.map_err(move |e| debug!("error TLS handshake: {:?} error: {:?}", src_addr, e))
                })
                .map_err(|e| debug!("error in inbound tcp_stream: {}", e)),
        );

        Ok(())
    }

    /// TODO: how to do threads? should we do a bunch of listener threads and then query threads?
    /// Ideally the processing would be n-threads for recieving, which hand off to m-threads for
    ///  request handling. It would generally be the case that n <= m.
    pub fn listen(&mut self) -> io::Result<()> {
        info!("Server starting up");
        self.io_loop.run(Forever)?;

        Err(io::Error::new(
            io::ErrorKind::Interrupted,
            "Server stopping due to interruption",
        ))
    }

    /// Returns a reference to the tokio core loop driving this Server instance
    pub fn tokio_core(&mut self) -> &mut Core {
        &mut self.io_loop
    }

    fn handle_request(
        request: Request,
        mut response_handle: ResponseHandle,
        handler: Arc<T>,
    ) -> io::Result<()> {
        info!(
            "request: {} type: {:?} op_code: {:?} dnssec: {} {}",
            request.message.id(),
            request.message.message_type(),
            request.message.op_code(),
            request
                .message
                .edns()
                .map_or(false, |edns| edns.dnssec_ok()),
            request
                .message
                .queries()
                .first()
                .map(|q| q.to_string())
                .unwrap_or_else(|| "empty_queries".to_string()),
        );
        let response = handler.handle_request(&request);

        info!(
            "request: {} response_code: {} answers: {} name_servers: {} additionals: {}",
            request.message.id(),
            response.response_code(),
            response.answers().len(),
            response.name_servers().len(),
            response.additionals().len(),
        );
        response_handle.send(response)
    }
}

struct Forever;

impl Future for Forever {
    type Item = ();
    type Error = io::Error;

    fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
        // run forever...
        Ok(Async::NotReady)
    }
}
<|endoftext|>


<commit_before>//! Ethereum DPT protocol implementation

extern crate etcommon_bigint as bigint;
extern crate etcommon_crypto as hash;
extern crate etcommon_rlp as rlp;
extern crate sha3;
extern crate secp256k1;
#[macro_use]
extern crate futures;
extern crate tokio_io;
extern crate tokio_core;
extern crate time;

mod proto;
mod message;
mod util;

use message::*;
use proto::{DPTCodec, DPTCodecMessage};
use futures::future;
use futures::{Poll, Async, StartSend, AsyncSink, Future, Stream, Sink};
use tokio_io::{AsyncRead, AsyncWrite};
use tokio_io::codec::{Framed, Encoder, Decoder};
use tokio_core::reactor::{Timeout, Handle};
use tokio_core::net::{UdpSocket, UdpFramed};
use std::net::{IpAddr, SocketAddr, Ipv4Addr, Ipv6Addr};
use std::io;
use bigint::{H256, H512};
use rlp::UntrustedRlp;
use hash::SECP256K1;
use secp256k1::key::{PublicKey, SecretKey};
use util::pk2id;

fn retain_mut<T, F>(vec: &mut Vec<T>, mut f: F)
    where F: FnMut(&mut T) -> bool
{
    let len = vec.len();
    let mut del = 0;
    {
        let v = &mut **vec;

        for i in 0..len {
            if !f(&mut v[i]) {
                del += 1;
            } else if del > 0 {
                v.swap(i - del, i);
            }
        }
    }
    if del > 0 {
        vec.truncate(len - del);
    }
}

/// DPT message for requesting new peers or ping with timeout
pub enum DPTMessage {
    RequestNewPeer,
    Ping(Timeout),
}

/// DPT stream for sending DPT messages or receiving new peers
pub struct DPTStream {
    stream: UdpFramed<DPTCodec>,
    id: H512,
    connected: Vec<DPTNode>,
    timeout: Option<(Timeout, Vec<H512>)>,
    incoming: Vec<DPTNode>,
    address: IpAddr,
    udp_port: u16,
    tcp_port: u16,
}

#[derive(Debug, Clone)]
/// DPT node used by a DPT stream
pub struct DPTNode {
    pub address: IpAddr,
    pub tcp_port: u16,
    pub udp_port: u16,
    pub id: H512,
}

impl DPTNode {
    /// The TCP socket address of this node
    pub fn tcp_addr(&self) -> SocketAddr {
        SocketAddr::new(self.address, self.tcp_port)
    }

    /// The UDP socket address of this node
    pub fn udp_addr(&self) -> SocketAddr {
        SocketAddr::new(self.address, self.udp_port)
    }
}

impl DPTStream {
    /// Create a new DPT stream
    pub fn new(addr: &SocketAddr, handle: &Handle,
               secret_key: SecretKey,
               bootstrap_nodes: Vec<DPTNode>,
               tcp_port: u16) -> Result<Self, io::Error> {
        let id = pk2id(&match PublicKey::from_secret_key(&SECP256K1, &secret_key) {
            Ok(val) => val,
            Err(_) => return Err(io::Error::new(io::ErrorKind::Other, "converting pub key failed")),
        });
        Ok(Self {
            stream: UdpSocket::bind(addr, handle)?.framed(DPTCodec::new(secret_key)),
            id, connected: bootstrap_nodes, incoming: Vec::new(),
            timeout: None,
            address: addr.ip(), udp_port: addr.port(), tcp_port
        })
    }
}

impl Stream for DPTStream {
    type Item = DPTNode;
    type Error = io::Error;

    fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
        let mut timeoutted = false;
        if self.timeout.is_some() {
            let &mut (ref mut timeout, ref hs) = self.timeout.as_mut().unwrap();
            timeoutted = match timeout.poll() {
                Ok(Async::Ready(())) => true,
                Ok(Async::NotReady) => false,
                Err(e) => return Err(e),
            };

            if timeoutted {
                for h in hs {
                    self.connected.retain(|v| v.id != *h);
                }
            }
        }
        if timeoutted {
            self.timeout = None;
        }

        let (message, remote_id, hash) = match try_ready!(self.stream.poll()) {
            Some(Some(val)) => val,
            Some(None) =>
                if self.incoming.len() > 0 {
                    return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                } else {
                    return Ok(Async::NotReady);
                },
            None => return Ok(Async::Ready(None)),
        };

        match message.typ {
            0x01 /* ping */ => {
                let incoming_message: PingMessage = match UntrustedRlp::new(&message.data).as_val() {
                    Ok(val) => val,
                    Err(_) =>
                        if self.incoming.len() > 0 {
                            return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                        } else {
                            return Ok(Async::NotReady);
                        },
                };
                let typ = 0x02u8;
                let echo = hash;
                let to = incoming_message.from;
                let outgoing_message = PongMessage {
                    echo, to, timestamp: time::now_utc().to_timespec().sec as u64,
                };
                let data = rlp::encode(&outgoing_message).to_vec();

                // If sending pong is not available, drop.
                self.stream.start_send(DPTCodecMessage {
                    typ, data, addr: message.addr
                });

                if self.incoming.len() > 0 {
                    return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                } else {
                    return Ok(Async::NotReady);
                }
            },
            0x02 /* pong */ => {
                let incoming_message: PongMessage = match UntrustedRlp::new(&message.data).as_val() {
                    Ok(val) => val,
                    Err(_) =>
                        if self.incoming.len() > 0 {
                            return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                        } else {
                            return Ok(Async::NotReady);
                        },
                };
                if self.timeout.is_some() {
                    self.timeout.as_mut().unwrap().1.retain(|v| {
                        *v != remote_id
                    });
                }

                if self.incoming.len() > 0 {
                    return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                } else {
                    return Ok(Async::NotReady);
                }
            },
            0x03 /* find neighbours */ => {
                // Return at most 3 nodes at a time.
                let mut nodes = Vec::new();
                for i in 0..self.connected.len() {
                    if nodes.len() >= 3 {
                        break;
                    }

                    let address = self.connected[i].address;
                    let udp_port = self.connected[i].udp_port;
                    let tcp_port = self.connected[i].tcp_port;
                    let id = self.connected[i].id;

                    nodes.push(Neighbour {
                        address, udp_port, tcp_port, id,
                    });
                }
                let typ = 0x04u8;
                let outgoing_message = NeighboursMessage {
                    nodes, timestamp: time::now_utc().to_timespec().sec as u64,
                };
                let data = rlp::encode(&outgoing_message).to_vec();

                self.stream.start_send(DPTCodecMessage {
                    typ, data, addr: message.addr
                });

                if self.incoming.len() > 0 {
                    return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                } else {
                    return Ok(Async::NotReady);
                }
            },
            0x04 /* neighbours */ => {
                let incoming_message: NeighboursMessage = match UntrustedRlp::new(&message.data).as_val() {
                    Ok(val) => val,
                    Err(_) =>
                        if self.incoming.len() > 0 {
                            return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                        } else {
                            return Ok(Async::NotReady);
                        }
                };
                for i in 0..incoming_message.nodes.len() {
                    if self.connected.iter()
                        .all(|node| node.id != incoming_message.nodes[i].id)
                    {
                        let ip = incoming_message.nodes[i].address;
                        let tcp_port = incoming_message.nodes[i].tcp_port;
                        let udp_port = incoming_message.nodes[i].udp_port;
                        let remote_id = incoming_message.nodes[i].id;
                        let addr = SocketAddr::new(ip, udp_port);
                        let node = DPTNode {
                            address: ip,
                            udp_port, tcp_port,
                            id: remote_id
                        };
                        self.connected.push(node.clone());
                        self.incoming.push(node);
                    }
                }
                if self.incoming.len() > 0 {
                    return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                } else {
                    return Ok(Async::NotReady);
                }
            },
            _ => {
                if self.incoming.len() > 0 {
                    return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                } else {
                    return Ok(Async::NotReady);
                }
            }
        }
    }
}

impl Sink for DPTStream {
    type SinkItem = DPTMessage;
    type SinkError = io::Error;

    fn poll_complete(&mut self) -> Poll<(), Self::SinkError> {
        self.stream.poll_complete()
    }

    fn start_send(&mut self, message: DPTMessage) -> StartSend<Self::SinkItem, Self::SinkError> {
        match message {

            DPTMessage::RequestNewPeer => {
                let mut any_sent = false;
                let typ = 0x03u8;
                let message = FindNeighboursMessage {
                    id: self.id,
                    timestamp: time::now_utc().to_timespec().sec as u64,
                };
                let data = rlp::encode(&message).to_vec();
                let ref mut connected = self.connected;
                let ref mut stream = self.stream;
                retain_mut(connected, |node| {
                    match stream.start_send(DPTCodecMessage {
                        addr: node.udp_addr(), typ, data: data.clone()
                    }) {
                        Ok(AsyncSink::Ready) => {
                            any_sent = true;
                            true
                        },
                        Ok(AsyncSink::NotReady(_)) => true,
                        Err(_) => false,
                    }
                });
                if any_sent {
                    return Ok(AsyncSink::Ready);
                } else {
                    return Ok(AsyncSink::NotReady(DPTMessage::RequestNewPeer));
                }
            },

            DPTMessage::Ping(timeout) => {
                let mut any_sent = false;
                let mut timeouting = Vec::new();
                let from_endpoint = Endpoint {
                    address: self.address.clone(),
                    udp_port: self.udp_port,
                    tcp_port: self.tcp_port,
                };
                let ref mut connected = self.connected;
                let ref mut stream = self.stream;
                retain_mut(connected, |node| {
                    let typ = 0x01u8;
                    let (address, tcp_port, udp_port, remote_id) =
                        (node.address, node.tcp_port, node.udp_port, node.id);
                    let message = PingMessage {
                        version: H256::from(0x3),
                        from: from_endpoint.clone(),
                        to: Endpoint {
                            address, udp_port, tcp_port
                        },
                        timestamp: time::now_utc().to_timespec().sec as u64,
                    };
                    let data = rlp::encode(&message).to_vec();
                    match stream.start_send(DPTCodecMessage {
                        addr: node.udp_addr(), typ, data
                    }) {
                        Ok(AsyncSink::Ready) => {
                            any_sent = true;
                            timeouting.push(remote_id);
                            true
                        },
                        Ok(AsyncSink::NotReady(_)) => true,
                        Err(_) => false,
                    }
                });
                if any_sent {
                    self.timeout = Some((timeout, timeouting));
                    return Ok(AsyncSink::Ready);
                } else {
                    return Ok(AsyncSink::NotReady(DPTMessage::Ping(timeout)));
                }
            }

        }
    }
}

#[cfg(test)]
mod tests {
    #[test]
    fn it_works() {
    }
}
<commit_msg>dpt: make no reachable endpoint as error<commit_after>//! Ethereum DPT protocol implementation

extern crate etcommon_bigint as bigint;
extern crate etcommon_crypto as hash;
extern crate etcommon_rlp as rlp;
extern crate sha3;
extern crate secp256k1;
#[macro_use]
extern crate futures;
extern crate tokio_io;
extern crate tokio_core;
extern crate time;

mod proto;
mod message;
mod util;

use message::*;
use proto::{DPTCodec, DPTCodecMessage};
use futures::future;
use futures::{Poll, Async, StartSend, AsyncSink, Future, Stream, Sink};
use tokio_io::{AsyncRead, AsyncWrite};
use tokio_io::codec::{Framed, Encoder, Decoder};
use tokio_core::reactor::{Timeout, Handle};
use tokio_core::net::{UdpSocket, UdpFramed};
use std::net::{IpAddr, SocketAddr, Ipv4Addr, Ipv6Addr};
use std::io;
use bigint::{H256, H512};
use rlp::UntrustedRlp;
use hash::SECP256K1;
use secp256k1::key::{PublicKey, SecretKey};
use util::pk2id;

fn retain_mut<T, F>(vec: &mut Vec<T>, mut f: F)
    where F: FnMut(&mut T) -> bool
{
    let len = vec.len();
    let mut del = 0;
    {
        let v = &mut **vec;

        for i in 0..len {
            if !f(&mut v[i]) {
                del += 1;
            } else if del > 0 {
                v.swap(i - del, i);
            }
        }
    }
    if del > 0 {
        vec.truncate(len - del);
    }
}

/// DPT message for requesting new peers or ping with timeout
pub enum DPTMessage {
    RequestNewPeer,
    Ping(Timeout),
}

/// DPT stream for sending DPT messages or receiving new peers
pub struct DPTStream {
    stream: UdpFramed<DPTCodec>,
    id: H512,
    connected: Vec<DPTNode>,
    timeout: Option<(Timeout, Vec<H512>)>,
    incoming: Vec<DPTNode>,
    address: IpAddr,
    udp_port: u16,
    tcp_port: u16,
}

#[derive(Debug, Clone)]
/// DPT node used by a DPT stream
pub struct DPTNode {
    pub address: IpAddr,
    pub tcp_port: u16,
    pub udp_port: u16,
    pub id: H512,
}

impl DPTNode {
    /// The TCP socket address of this node
    pub fn tcp_addr(&self) -> SocketAddr {
        SocketAddr::new(self.address, self.tcp_port)
    }

    /// The UDP socket address of this node
    pub fn udp_addr(&self) -> SocketAddr {
        SocketAddr::new(self.address, self.udp_port)
    }
}

impl DPTStream {
    /// Create a new DPT stream
    pub fn new(addr: &SocketAddr, handle: &Handle,
               secret_key: SecretKey,
               bootstrap_nodes: Vec<DPTNode>,
               tcp_port: u16) -> Result<Self, io::Error> {
        let id = pk2id(&match PublicKey::from_secret_key(&SECP256K1, &secret_key) {
            Ok(val) => val,
            Err(_) => return Err(io::Error::new(io::ErrorKind::Other, "converting pub key failed")),
        });
        Ok(Self {
            stream: UdpSocket::bind(addr, handle)?.framed(DPTCodec::new(secret_key)),
            id, connected: bootstrap_nodes, incoming: Vec::new(),
            timeout: None,
            address: addr.ip(), udp_port: addr.port(), tcp_port
        })
    }
}

impl Stream for DPTStream {
    type Item = DPTNode;
    type Error = io::Error;

    fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
        let mut timeoutted = false;
        if self.timeout.is_some() {
            let &mut (ref mut timeout, ref hs) = self.timeout.as_mut().unwrap();
            timeoutted = match timeout.poll() {
                Ok(Async::Ready(())) => true,
                Ok(Async::NotReady) => false,
                Err(e) => return Err(e),
            };

            if timeoutted {
                for h in hs {
                    self.connected.retain(|v| v.id != *h);
                }
            }
        }
        if timeoutted {
            self.timeout = None;
        }

        let (message, remote_id, hash) = match try!(self.stream.poll()) {
            Async::Ready(Some(Some(val))) => val,
            Async::Ready(Some(None)) | Async::NotReady => {
                if self.incoming.len() > 0 {
                    return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                } else if self.connected.len() == 0 {
                    return Err(io::Error::new(io::ErrorKind::Other, "no reachable DPT endpoint"));
                } else {
                    return Ok(Async::NotReady);
                }
            },
            Async::Ready(None) => return Ok(Async::Ready(None)),
        };

        match message.typ {
            0x01 /* ping */ => {
                let incoming_message: PingMessage = match UntrustedRlp::new(&message.data).as_val() {
                    Ok(val) => val,
                    Err(_) =>
                        if self.incoming.len() > 0 {
                            return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                        } else {
                            return Ok(Async::NotReady);
                        },
                };
                let typ = 0x02u8;
                let echo = hash;
                let to = incoming_message.from;
                let outgoing_message = PongMessage {
                    echo, to, timestamp: time::now_utc().to_timespec().sec as u64,
                };
                let data = rlp::encode(&outgoing_message).to_vec();

                // If sending pong is not available, drop.
                self.stream.start_send(DPTCodecMessage {
                    typ, data, addr: message.addr
                });

                if self.incoming.len() > 0 {
                    return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                } else {
                    return Ok(Async::NotReady);
                }
            },
            0x02 /* pong */ => {
                let incoming_message: PongMessage = match UntrustedRlp::new(&message.data).as_val() {
                    Ok(val) => val,
                    Err(_) =>
                        if self.incoming.len() > 0 {
                            return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                        } else {
                            return Ok(Async::NotReady);
                        },
                };
                if self.timeout.is_some() {
                    self.timeout.as_mut().unwrap().1.retain(|v| {
                        *v != remote_id
                    });
                }

                if self.incoming.len() > 0 {
                    return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                } else {
                    return Ok(Async::NotReady);
                }
            },
            0x03 /* find neighbours */ => {
                // Return at most 3 nodes at a time.
                let mut nodes = Vec::new();
                for i in 0..self.connected.len() {
                    if nodes.len() >= 3 {
                        break;
                    }

                    let address = self.connected[i].address;
                    let udp_port = self.connected[i].udp_port;
                    let tcp_port = self.connected[i].tcp_port;
                    let id = self.connected[i].id;

                    nodes.push(Neighbour {
                        address, udp_port, tcp_port, id,
                    });
                }
                let typ = 0x04u8;
                let outgoing_message = NeighboursMessage {
                    nodes, timestamp: time::now_utc().to_timespec().sec as u64,
                };
                let data = rlp::encode(&outgoing_message).to_vec();

                self.stream.start_send(DPTCodecMessage {
                    typ, data, addr: message.addr
                });

                if self.incoming.len() > 0 {
                    return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                } else {
                    return Ok(Async::NotReady);
                }
            },
            0x04 /* neighbours */ => {
                let incoming_message: NeighboursMessage = match UntrustedRlp::new(&message.data).as_val() {
                    Ok(val) => val,
                    Err(_) =>
                        if self.incoming.len() > 0 {
                            return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                        } else {
                            return Ok(Async::NotReady);
                        }
                };
                for i in 0..incoming_message.nodes.len() {
                    if self.connected.iter()
                        .all(|node| node.id != incoming_message.nodes[i].id)
                    {
                        let ip = incoming_message.nodes[i].address;
                        let tcp_port = incoming_message.nodes[i].tcp_port;
                        let udp_port = incoming_message.nodes[i].udp_port;
                        let remote_id = incoming_message.nodes[i].id;
                        let addr = SocketAddr::new(ip, udp_port);
                        let node = DPTNode {
                            address: ip,
                            udp_port, tcp_port,
                            id: remote_id
                        };
                        self.connected.push(node.clone());
                        self.incoming.push(node);
                    }
                }
                if self.incoming.len() > 0 {
                    return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                } else {
                    return Ok(Async::NotReady);
                }
            },
            _ => {
                if self.incoming.len() > 0 {
                    return Ok(Async::Ready(Some(self.incoming.pop().unwrap())));
                } else {
                    return Ok(Async::NotReady);
                }
            }
        }
    }
}

impl Sink for DPTStream {
    type SinkItem = DPTMessage;
    type SinkError = io::Error;

    fn poll_complete(&mut self) -> Poll<(), Self::SinkError> {
        self.stream.poll_complete()
    }

    fn start_send(&mut self, message: DPTMessage) -> StartSend<Self::SinkItem, Self::SinkError> {
        match message {

            DPTMessage::RequestNewPeer => {
                let mut any_sent = false;
                let typ = 0x03u8;
                let message = FindNeighboursMessage {
                    id: self.id,
                    timestamp: time::now_utc().to_timespec().sec as u64,
                };
                let data = rlp::encode(&message).to_vec();
                let ref mut connected = self.connected;
                let ref mut stream = self.stream;
                retain_mut(connected, |node| {
                    match stream.start_send(DPTCodecMessage {
                        addr: node.udp_addr(), typ, data: data.clone()
                    }) {
                        Ok(AsyncSink::Ready) => {
                            any_sent = true;
                            true
                        },
                        Ok(AsyncSink::NotReady(_)) => true,
                        Err(_) => false,
                    }
                });
                if any_sent {
                    return Ok(AsyncSink::Ready);
                } else {
                    return Ok(AsyncSink::NotReady(DPTMessage::RequestNewPeer));
                }
            },

            DPTMessage::Ping(timeout) => {
                let mut any_sent = false;
                let mut timeouting = Vec::new();
                let from_endpoint = Endpoint {
                    address: self.address.clone(),
                    udp_port: self.udp_port,
                    tcp_port: self.tcp_port,
                };
                let ref mut connected = self.connected;
                let ref mut stream = self.stream;
                retain_mut(connected, |node| {
                    let typ = 0x01u8;
                    let (address, tcp_port, udp_port, remote_id) =
                        (node.address, node.tcp_port, node.udp_port, node.id);
                    let message = PingMessage {
                        version: H256::from(0x3),
                        from: from_endpoint.clone(),
                        to: Endpoint {
                            address, udp_port, tcp_port
                        },
                        timestamp: time::now_utc().to_timespec().sec as u64,
                    };
                    let data = rlp::encode(&message).to_vec();
                    match stream.start_send(DPTCodecMessage {
                        addr: node.udp_addr(), typ, data
                    }) {
                        Ok(AsyncSink::Ready) => {
                            any_sent = true;
                            timeouting.push(remote_id);
                            true
                        },
                        Ok(AsyncSink::NotReady(_)) => true,
                        Err(_) => false,
                    }
                });
                if any_sent {
                    self.timeout = Some((timeout, timeouting));
                    return Ok(AsyncSink::Ready);
                } else {
                    return Ok(AsyncSink::NotReady(DPTMessage::Ping(timeout)));
                }
            }

        }
    }
}

#[cfg(test)]
mod tests {
    #[test]
    fn it_works() {
    }
}
<|endoftext|>


<commit_before>use std::io::{Write, Result as IOResult};

use ansi_term::ANSIStrings;
use term_grid as grid;

use fs::{Dir, File};
use fs::feature::xattr::FileAttributes;
use fs::filter::FileFilter;

use output::cell::TextCell;
use output::colours::Colours;
use output::details::{Options as DetailsOptions, Row as DetailsRow, Render as DetailsRender};
use output::grid::Options as GridOptions;
use output::file_name::FileStyle;
use output::table::{Table, Row as TableRow, Options as TableOptions};
use output::tree::{TreeParams, TreeDepth};

pub struct Render<'a> {
    pub dir: Option<&'a Dir>,
    pub files: Vec<File<'a>>,
    pub colours: &'a Colours,
    pub style: &'a FileStyle,
    pub grid: &'a GridOptions,
    pub details: &'a DetailsOptions,
    pub filter: &'a FileFilter,
}

impl<'a> Render<'a> {
    pub fn details(&self) -> DetailsRender<'a> {
        DetailsRender {
            dir: self.dir.clone(),
            files: Vec::new(),
            colours: self.colours,
            style: self.style,
            opts: self.details,
            recurse: None,
            filter: self.filter,
        }
    }

    pub fn render<W: Write>(&self, w: &mut W) -> IOResult<()> {

        let options = self.details.table.as_ref().expect("Details table options not given!");

        let drender = self.clone().details();

        let (first_table, _) = self.make_table(options, &drender);

        let rows = self.files.iter()
                       .map(|file| first_table.row_for_file(file, file_has_xattrs(file)))
                       .collect::<Vec<TableRow>>();

        let file_names = self.files.iter()
                             .map(|file| self.style.for_file(file, self.colours).paint().promote())
                             .collect::<Vec<TextCell>>();

        let mut last_working_table = self.make_grid(1, options, &file_names, rows.clone(), &drender);

        for column_count in 2.. {
            let grid = self.make_grid(column_count, options, &file_names, rows.clone(), &drender);

            let the_grid_fits = {
                let d = grid.fit_into_columns(column_count);
                d.is_complete() && d.width() <= self.grid.console_width
            };

            if the_grid_fits {
                last_working_table = grid;
            }
            else {
                return write!(w, "{}", last_working_table.fit_into_columns(column_count - 1));
            }
        }

        Ok(())
    }

    fn make_table<'t>(&'a self, options: &'a TableOptions, drender: &DetailsRender) -> (Table<'a>, Vec<DetailsRow>) {
        let mut table = Table::new(options, self.dir, self.colours);
        let mut rows = Vec::new();

        if self.details.header {
            let row = table.header_row();
            table.add_widths(&row);
            rows.push(drender.render_header(row));
        }

        (table, rows)
    }

    fn make_grid(&'a self, column_count: usize, options: &'a TableOptions, file_names: &[TextCell], rows: Vec<TableRow>, drender: &DetailsRender) -> grid::Grid {

        let mut tables = Vec::new();
        for _ in 0 .. column_count {
            tables.push(self.make_table(options, drender));
        }

        let mut num_cells = rows.len();
        if self.details.header {
            num_cells += column_count;
        }

        let original_height = divide_rounding_up(rows.len(), column_count);
        let height = divide_rounding_up(num_cells, column_count);

        for (i, (file_name, row)) in file_names.iter().zip(rows.into_iter()).enumerate() {
            let index = if self.grid.across {
                    i % column_count
                }
                else {
                    i / original_height
                };

            let (ref mut table, ref mut rows) = tables[index];
            table.add_widths(&row);
            let details_row = drender.render_file(row, file_name.clone(), TreeParams::new(TreeDepth::root(), false));
            rows.push(details_row);
        }

        let columns: Vec<_> = tables.into_iter().map(|(table, details_rows)| {
            drender.iterate_with_table(table, details_rows).collect::<Vec<_>>()
        }).collect();

        let direction = if self.grid.across { grid::Direction::LeftToRight }
                                       else { grid::Direction::TopToBottom };

        let mut grid = grid::Grid::new(grid::GridOptions {
            direction:  direction,
            filling:    grid::Filling::Spaces(4),
        });

        if self.grid.across {
            for row in 0 .. height {
                for column in &columns {
                    if row < column.len() {
                        let cell = grid::Cell {
                            contents: ANSIStrings(&column[row].contents).to_string(),
                            width:    *column[row].width,
                        };

                        grid.add(cell);
                    }
                }
            }
        }
        else {
            for column in &columns {
                for cell in column.iter() {
                    let cell = grid::Cell {
                        contents: ANSIStrings(&cell.contents).to_string(),
                        width:    *cell.width,
                    };

                    grid.add(cell);
                }
            }
        }

        grid
    }
}

fn divide_rounding_up(a: usize, b: usize) -> usize {
    let mut result = a / b;
    if a % b != 0 { result += 1; }
    result
}

fn file_has_xattrs(file: &File) -> bool {
    match file.path.attributes() {
        Ok(attrs) => !attrs.is_empty(),
        Err(_) => false,
    }
}
<commit_msg>Remove unnecessary clone (plus docs)<commit_after>//! The grid-details view lists several details views side-by-side.

use std::io::{Write, Result as IOResult};

use ansi_term::ANSIStrings;
use term_grid as grid;

use fs::{Dir, File};
use fs::feature::xattr::FileAttributes;
use fs::filter::FileFilter;

use output::cell::TextCell;
use output::colours::Colours;
use output::details::{Options as DetailsOptions, Row as DetailsRow, Render as DetailsRender};
use output::grid::Options as GridOptions;
use output::file_name::FileStyle;
use output::table::{Table, Row as TableRow, Options as TableOptions};
use output::tree::{TreeParams, TreeDepth};

pub struct Render<'a> {

    /// The directory that’s being rendered here.
    /// We need this to know which columns to put in the output.
    pub dir: Option<&'a Dir>,

    /// The files that have been read from the directory. They should all
    /// hold a reference to it.
    pub files: Vec<File<'a>>,

    /// How to colour various pieces of text.
    pub colours: &'a Colours,

    /// How to format filenames.
    pub style: &'a FileStyle,

    /// The grid part of the grid-details view.
    pub grid: &'a GridOptions,

    /// The details part of the grid-details view.
    pub details: &'a DetailsOptions,

    /// How to filter files after listing a directory. The files in this
    /// render will already have been filtered and sorted, but any directories
    /// that we recurse into will have to have this applied.
    pub filter: &'a FileFilter,
}

impl<'a> Render<'a> {
    pub fn details(&self) -> DetailsRender<'a> {
        DetailsRender {
            dir: self.dir.clone(),
            files: Vec::new(),
            colours: self.colours,
            style: self.style,
            opts: self.details,
            recurse: None,
            filter: self.filter,
        }
    }

    pub fn render<W: Write>(&self, w: &mut W) -> IOResult<()> {

        let options = self.details.table.as_ref().expect("Details table options not given!");

        let drender = self.details();

        let (first_table, _) = self.make_table(options, &drender);

        let rows = self.files.iter()
                       .map(|file| first_table.row_for_file(file, file_has_xattrs(file)))
                       .collect::<Vec<TableRow>>();

        let file_names = self.files.iter()
                             .map(|file| self.style.for_file(file, self.colours).paint().promote())
                             .collect::<Vec<TextCell>>();

        let mut last_working_table = self.make_grid(1, options, &file_names, rows.clone(), &drender);

        for column_count in 2.. {
            let grid = self.make_grid(column_count, options, &file_names, rows.clone(), &drender);

            let the_grid_fits = {
                let d = grid.fit_into_columns(column_count);
                d.is_complete() && d.width() <= self.grid.console_width
            };

            if the_grid_fits {
                last_working_table = grid;
            }
            else {
                return write!(w, "{}", last_working_table.fit_into_columns(column_count - 1));
            }
        }

        Ok(())
    }

    fn make_table<'t>(&'a self, options: &'a TableOptions, drender: &DetailsRender) -> (Table<'a>, Vec<DetailsRow>) {
        let mut table = Table::new(options, self.dir, self.colours);
        let mut rows = Vec::new();

        if self.details.header {
            let row = table.header_row();
            table.add_widths(&row);
            rows.push(drender.render_header(row));
        }

        (table, rows)
    }

    fn make_grid(&'a self, column_count: usize, options: &'a TableOptions, file_names: &[TextCell], rows: Vec<TableRow>, drender: &DetailsRender) -> grid::Grid {

        let mut tables = Vec::new();
        for _ in 0 .. column_count {
            tables.push(self.make_table(options, drender));
        }

        let mut num_cells = rows.len();
        if self.details.header {
            num_cells += column_count;
        }

        let original_height = divide_rounding_up(rows.len(), column_count);
        let height = divide_rounding_up(num_cells, column_count);

        for (i, (file_name, row)) in file_names.iter().zip(rows.into_iter()).enumerate() {
            let index = if self.grid.across {
                    i % column_count
                }
                else {
                    i / original_height
                };

            let (ref mut table, ref mut rows) = tables[index];
            table.add_widths(&row);
            let details_row = drender.render_file(row, file_name.clone(), TreeParams::new(TreeDepth::root(), false));
            rows.push(details_row);
        }

        let columns: Vec<_> = tables.into_iter().map(|(table, details_rows)| {
            drender.iterate_with_table(table, details_rows).collect::<Vec<_>>()
        }).collect();

        let direction = if self.grid.across { grid::Direction::LeftToRight }
                                       else { grid::Direction::TopToBottom };

        let mut grid = grid::Grid::new(grid::GridOptions {
            direction:  direction,
            filling:    grid::Filling::Spaces(4),
        });

        if self.grid.across {
            for row in 0 .. height {
                for column in &columns {
                    if row < column.len() {
                        let cell = grid::Cell {
                            contents: ANSIStrings(&column[row].contents).to_string(),
                            width:    *column[row].width,
                        };

                        grid.add(cell);
                    }
                }
            }
        }
        else {
            for column in &columns {
                for cell in column.iter() {
                    let cell = grid::Cell {
                        contents: ANSIStrings(&cell.contents).to_string(),
                        width:    *cell.width,
                    };

                    grid.add(cell);
                }
            }
        }

        grid
    }
}

fn divide_rounding_up(a: usize, b: usize) -> usize {
    let mut result = a / b;
    if a % b != 0 { result += 1; }
    result
}

fn file_has_xattrs(file: &File) -> bool {
    match file.path.attributes() {
        Ok(attrs) => !attrs.is_empty(),
        Err(_) => false,
    }
}
<|endoftext|>