<commit_before>// This example implements an OCR (optical character recognition):
// https://en.wikipedia.org/wiki/Optical_character_recognition
// using an evolutionary algorithm.
//
// Note that evolutionary algorithms do no guarantee to always find the optimal solution.
// But they can get very close.

extern crate rand;
#[macro_use] extern crate log;
extern crate image;
extern crate imageproc;
extern crate rusttype;
extern crate simplelog;

// internal crates
extern crate darwin_rs;

use std::sync::Arc;
use rand::Rng;
use std::fs::File;
use std::io::Read;
use std::path::Path;
use image::{ImageBuffer, Luma};
use imageproc::stats::root_mean_squared_error;
use simplelog::{SimpleLogger, LogLevelFilter};
use std::str;

// internal modules
use darwin_rs::{Individual, SimulationBuilder, PopulationBuilder, SimError};

fn make_population<'a>(count: u32, config: &OCRConfig<'a>) -> Vec<OCRItem<'a>> {
    let mut result = Vec::new();

    let shared = Arc::new(config.clone());

    for _ in 0..count {
        result.push( OCRItem {
            content: vec![
                // Start with letter 'A' in each line
                TextBox{ x: 10, y: 10, text: vec![65] },
                TextBox{ x: 10, y: 40, text: vec![65] }],
            config: shared.clone()
        });
    }

    result
}

#[derive(Clone)]
struct TextBox {
    x: u32,
    y: u32,
    text: Vec<u8>
}

#[derive(Clone)]
struct OCRConfig<'a> {
    font: rusttype::Font<'a>,
    original_img: ImageBuffer<Luma<u8>, Vec<u8>>
}

#[derive(Clone)]
struct OCRItem<'a> {
    content: Vec<TextBox>,
    config: Arc<OCRConfig<'a>>
}

impl<'a> Individual for OCRItem<'a> {
    fn mutate(&mut self) {
        let mut rng = rand::thread_rng();

        let content_line = rng.gen_range(0, 2);

        let operation = rng.gen_range(0, 4);

        let index1 = rng.gen_range(0, self.content[content_line].text.len());

        match operation {
            0 => {
                // Add character
                let new_char = rng.gen_range(32, 127); // All printable ASCII characters
                self.content[content_line].text.insert(index1, new_char);
            },
            1 => {
                // Remove character
                if self.content[content_line].text.len() > 1 {
                    // Leave at least one character
                    self.content[content_line].text.remove(index1);
                }
            },
            2 => {
                // Change character
                let new_char = rng.gen_range(32, 127); // All printable ASCII characters
                self.content[content_line].text[index1] = new_char;
            },
            3 => {
                // Swap characters
                let index2 = rng.gen_range(0, self.content[content_line].text.len());
                let temp = self.content[content_line].text[index1];
                self.content[content_line].text[index1] = self.content[content_line].text[index2];
                self.content[content_line].text[index2] = temp;
            },
            4 => {
                // You can think of more operations: shift / rotate, mirror, ...
            }
            n => info!("mutate(): unknown operation: {}", n)
        }
    }

    fn calculate_fitness(&mut self) -> f64 {
        let mut constructed_img: ImageBuffer<Luma<u8>, Vec<u8>> = ImageBuffer::new(120, 70);

        draw_text_line(&mut constructed_img, &self.config.font,
            self.content[0].x as i32, self.content[0].y as i32,
            str::from_utf8(&self.content[0].text).unwrap());

        draw_text_line(&mut constructed_img, &self.config.font,
            self.content[1].x as i32, self.content[1].y as i32,
            str::from_utf8(&self.content[1].text).unwrap());

        root_mean_squared_error(&self.config.original_img, &constructed_img)
    }

    fn reset(&mut self) {
        self.content = vec![
        TextBox{ x: 10, y: 10, text: vec![65] },
        TextBox{ x: 10, y: 40, text: vec![65] }];
    }
}

fn draw_text_line(canvas: &mut ImageBuffer<Luma<u8>, Vec<u8>>,
    font: &rusttype::Font, pos_x: i32, pos_y: i32, text: &str) {

    let height: f32 = 18.0;
    let scale = rusttype::Scale { x: height * 1.0, y: height };
    let v_metrics = font.v_metrics(scale);
    let offset = rusttype::point(0.0, v_metrics.ascent);
    let glyphs: Vec<rusttype::PositionedGlyph> = font.layout(text, scale, offset).collect();

    for g in glyphs {
        if let Some(bb) = g.pixel_bounding_box() {
            g.draw(|x, y, v| {
                let x = (x as i32) + bb.min.x + pos_x;
                let y = (y as i32) + bb.min.y + pos_y;
                if x >=0 && y >= 0 && x < canvas.width() as i32 && y < canvas.height() as i32 {
                    canvas.put_pixel(x as u32, y as u32, Luma::<u8>{ data: [(v * 255.0) as u8] } );
                }
            })
        }
    }

}

fn main() {
    println!("Darwin test: optical character recognition");

    let _ = SimpleLogger::init(LogLevelFilter::Info);

    // TODO: use fontconfig-rs in the future: https://github.com/abonander/fontconfig-rs
    let mut file = File::open("/usr/share/fonts/truetype/liberation/LiberationMono-Regular.ttf").unwrap();
    let mut font_data: Vec<u8> = Vec::new();
    let _ = file.read_to_end(&mut font_data).unwrap();

    let collection = rusttype::FontCollection::from_bytes(font_data);
    let font = collection.into_font().unwrap();

    let mut original_img: ImageBuffer<Luma<u8>, Vec<u8>> = ImageBuffer::new(120, 70);
    draw_text_line(&mut original_img, &font, 10, 10, "Darwin-rs");
    draw_text_line(&mut original_img, &font, 10, 40, "OCR test!");

    let img_file = Path::new("rendered_text.png");
    let _ = original_img.save(&img_file);

    let ocr_config = OCRConfig { font: font, original_img: original_img };

    let initial_population = make_population(50, &ocr_config);

    let population1 = PopulationBuilder::<OCRItem>::new()
        .set_id(1)
        .initial_population(&initial_population)
        .increasing_exp_mutation_rate(1.02)
        .reset_limit_increment(100)
        .reset_limit_start(100)
        .reset_limit_end(5000)
        .finalize().unwrap();

    let population2 = PopulationBuilder::<OCRItem>::new()
        .set_id(2)
        .initial_population(&initial_population)
        .increasing_exp_mutation_rate(1.04)
        .reset_limit_increment(200)
        .reset_limit_start(100)
        .reset_limit_end(5000)
        .finalize().unwrap();

    let population3 = PopulationBuilder::<OCRItem>::new()
        .set_id(3)
        .initial_population(&initial_population)
        .increasing_exp_mutation_rate(1.06)
        .reset_limit_increment(300)
        .reset_limit_start(100)
        .reset_limit_end(5000)
        .finalize().unwrap();

    let ocr_builder = SimulationBuilder::<OCRItem>::new()
        .fitness(0.0)
        .threads(2)
        .add_population(population1)
        .add_population(population2)
        .add_population(population3)
        .finalize();

    match ocr_builder {
        Err(SimError::EndIterationTooLow) => println!("more than 10 iteratons needed"),
        Ok(mut ocr_simulation) => {
            ocr_simulation.run();

            println!("total run time: {} ms", ocr_simulation.total_time_in_ms);
            println!("improvement factor: {}", ocr_simulation.simulation_result.improvement_factor);
            println!("number of iterations: {}", ocr_simulation.simulation_result.iteration_counter);

            let ref item = ocr_simulation.simulation_result.fittest[0].individual;
            let line1 = str::from_utf8(&item.content[0].text).unwrap();
            let line2 = str::from_utf8(&item.content[1].text).unwrap();
            println!("line1: {}, line2: {}", line1, line2);
        }
    }
}
<commit_msg>make OCR example simpler<commit_after>// This example implements an OCR (optical character recognition):
// https://en.wikipedia.org/wiki/Optical_character_recognition
// using an evolutionary algorithm.
//
// Note that evolutionary algorithms do no guarantee to always find the optimal solution.
// But they can get very close.

extern crate rand;
#[macro_use] extern crate log;
extern crate image;
extern crate imageproc;
extern crate rusttype;
extern crate simplelog;

// internal crates
extern crate darwin_rs;

use std::sync::Arc;
use rand::Rng;
use std::fs::File;
use std::io::Read;
use std::path::Path;
use image::{ImageBuffer, Luma};
use imageproc::stats::root_mean_squared_error;
use simplelog::{SimpleLogger, LogLevelFilter};
use std::str;

// internal modules
use darwin_rs::{Individual, SimulationBuilder, PopulationBuilder, SimError};

fn make_population<'a>(count: u32, config: &OCRConfig<'a>) -> Vec<OCRItem<'a>> {
    let mut result = Vec::new();

    let shared = Arc::new(config.clone());

    for _ in 0..count {
        result.push( OCRItem {
            content: vec![
                // Start with letter 'A' in each line
                TextBox{ x: 10, y: 10, text: vec![65, 65, 65, 65, 65, 65, 65, 65, 65] },
                TextBox{ x: 10, y: 40, text: vec![65, 65, 65, 65, 65, 65, 65, 65, 65] }],
            config: shared.clone()
        });
    }

    result
}

#[derive(Clone)]
struct TextBox {
    x: u32,
    y: u32,
    text: Vec<u8>
}

#[derive(Clone)]
struct OCRConfig<'a> {
    font: rusttype::Font<'a>,
    original_img: ImageBuffer<Luma<u8>, Vec<u8>>
}

#[derive(Clone)]
struct OCRItem<'a> {
    content: Vec<TextBox>,
    config: Arc<OCRConfig<'a>>
}

impl<'a> Individual for OCRItem<'a> {
    fn mutate(&mut self) {
        let mut rng = rand::thread_rng();

        let content_line = rng.gen_range(0, 2);

        let operation = rng.gen_range(0, 2);

        let index1 = rng.gen_range(0, self.content[content_line].text.len());

        match operation {
            0 => {
                // Change character
                let new_char = rng.gen_range(32, 127); // All printable ASCII characters
                self.content[content_line].text[index1] = new_char;
            },
            1 => {
                // Swap characters
                let index2 = rng.gen_range(0, self.content[content_line].text.len());
                let temp = self.content[content_line].text[index1];
                self.content[content_line].text[index1] = self.content[content_line].text[index2];
                self.content[content_line].text[index2] = temp;
            },
            2 => {
                // Add character
                let new_char = rng.gen_range(32, 127); // All printable ASCII characters
                self.content[content_line].text.insert(index1, new_char);
            },
            3 => {
                // Remove character
                if self.content[content_line].text.len() > 1 {
                    // Leave at least one character
                    self.content[content_line].text.remove(index1);
                }
            },
            4 => {
                // You can think of more operations: shift / rotate, mirror, ...
            }
            n => info!("mutate(): unknown operation: {}", n)
        }
    }

    fn calculate_fitness(&mut self) -> f64 {
        let mut constructed_img: ImageBuffer<Luma<u8>, Vec<u8>> = ImageBuffer::new(120, 70);

        draw_text_line(&mut constructed_img, &self.config.font,
            self.content[0].x as i32, self.content[0].y as i32,
            str::from_utf8(&self.content[0].text).unwrap());

        draw_text_line(&mut constructed_img, &self.config.font,
            self.content[1].x as i32, self.content[1].y as i32,
            str::from_utf8(&self.content[1].text).unwrap());

        root_mean_squared_error(&self.config.original_img, &constructed_img)
    }

    fn reset(&mut self) {
        self.content = vec![
        TextBox{ x: 10, y: 10, text: vec![65, 65, 65, 65, 65, 65, 65, 65, 65] },
        TextBox{ x: 10, y: 40, text: vec![65, 65, 65, 65, 65, 65, 65, 65, 65] }];
    }
}

fn draw_text_line(canvas: &mut ImageBuffer<Luma<u8>, Vec<u8>>,
    font: &rusttype::Font, pos_x: i32, pos_y: i32, text: &str) {

    let height: f32 = 18.0;
    let scale = rusttype::Scale { x: height * 1.0, y: height };
    let v_metrics = font.v_metrics(scale);
    let offset = rusttype::point(0.0, v_metrics.ascent);
    let glyphs: Vec<rusttype::PositionedGlyph> = font.layout(text, scale, offset).collect();

    for g in glyphs {
        if let Some(bb) = g.pixel_bounding_box() {
            g.draw(|x, y, v| {
                let x = (x as i32) + bb.min.x + pos_x;
                let y = (y as i32) + bb.min.y + pos_y;
                if x >=0 && y >= 0 && x < canvas.width() as i32 && y < canvas.height() as i32 {
                    canvas.put_pixel(x as u32, y as u32, Luma::<u8>{ data: [(v * 255.0) as u8] } );
                }
            })
        }
    }

}

fn main() {
    println!("Darwin test: optical character recognition");

    let _ = SimpleLogger::init(LogLevelFilter::Info);

    // TODO: use fontconfig-rs in the future: https://github.com/abonander/fontconfig-rs
    let mut file = File::open("/usr/share/fonts/truetype/liberation/LiberationMono-Regular.ttf").unwrap();
    let mut font_data: Vec<u8> = Vec::new();
    let _ = file.read_to_end(&mut font_data).unwrap();

    let collection = rusttype::FontCollection::from_bytes(font_data);
    let font = collection.into_font().unwrap();

    let mut original_img: ImageBuffer<Luma<u8>, Vec<u8>> = ImageBuffer::new(120, 70);
    draw_text_line(&mut original_img, &font, 10, 10, "Darwin-rs");
    draw_text_line(&mut original_img, &font, 10, 40, "OCR Test!");

    let img_file = Path::new("rendered_text.png");
    let _ = original_img.save(&img_file);

    let ocr_config = OCRConfig { font: font, original_img: original_img };

    let initial_population = make_population(100, &ocr_config);

    let population1 = PopulationBuilder::<OCRItem>::new()
        .set_id(1)
        .initial_population(&initial_population)
        .increasing_exp_mutation_rate(1.02)
        .reset_limit_increment(100)
        .reset_limit_start(100)
        .reset_limit_end(1000)
        .finalize().unwrap();

    let population2 = PopulationBuilder::<OCRItem>::new()
        .set_id(2)
        .initial_population(&initial_population)
        .increasing_exp_mutation_rate(1.04)
        .reset_limit_increment(200)
        .reset_limit_start(200)
        .reset_limit_end(2000)
        .finalize().unwrap();

    let population3 = PopulationBuilder::<OCRItem>::new()
        .set_id(3)
        .initial_population(&initial_population)
        .increasing_exp_mutation_rate(1.06)
        .reset_limit_increment(300)
        .reset_limit_start(300)
        .reset_limit_end(3000)
        .finalize().unwrap();

    let population4 = PopulationBuilder::<OCRItem>::new()
        .set_id(4)
        .initial_population(&initial_population)
        .increasing_exp_mutation_rate(1.08)
        .reset_limit_increment(400)
        .reset_limit_start(400)
        .reset_limit_end(4000)
        .finalize().unwrap();

    let population5 = PopulationBuilder::<OCRItem>::new()
        .set_id(5)
        .initial_population(&initial_population)
        .increasing_exp_mutation_rate(1.10)
        .reset_limit_increment(500)
        .reset_limit_start(500)
        .reset_limit_end(5000)
        .finalize().unwrap();

    let ocr_builder = SimulationBuilder::<OCRItem>::new()
        .fitness(0.0)
        .threads(2)
        .add_population(population1)
        .add_population(population2)
        .add_population(population3)
        .add_population(population4)
        .add_population(population5)
        .finalize();

    match ocr_builder {
        Err(SimError::EndIterationTooLow) => println!("more than 10 iteratons needed"),
        Ok(mut ocr_simulation) => {
            ocr_simulation.run();

            println!("total run time: {} ms", ocr_simulation.total_time_in_ms);
            println!("improvement factor: {}", ocr_simulation.simulation_result.improvement_factor);
            println!("number of iterations: {}", ocr_simulation.simulation_result.iteration_counter);

            let ref item = ocr_simulation.simulation_result.fittest[0].individual;
            let line1 = str::from_utf8(&item.content[0].text).unwrap();
            let line2 = str::from_utf8(&item.content[1].text).unwrap();
            println!("line1: {}, line2: {}", line1, line2);
        }
    }
}
<|endoftext|>


<commit_before>use proc_macro2::{Span, TokenStream};
use syn::{Data, Field, Fields, Ident, Lit, Meta, NestedMeta, Variant};

use super::RustlerAttr;

///
/// A parsing context struct.
///
/// `Context` holds information usable for different codegen modules.
///
pub(crate) struct Context<'a> {
    pub attrs: Vec<RustlerAttr>,
    pub ident: &'a proc_macro2::Ident,
    pub ident_with_lifetime: proc_macro2::TokenStream,
    pub variants: Option<Vec<&'a Variant>>,
    pub struct_fields: Option<Vec<&'a Field>>,
    pub is_tuple_struct: bool,
}

impl<'a> Context<'a> {
    pub fn from_ast(ast: &'a syn::DeriveInput) -> Self {
        let mut attrs: Vec<_> = ast
            .attrs
            .iter()
            .map(Context::get_rustler_attrs)
            .flatten()
            .collect();

        //
        // Default: generate encoder and decoder
        //
        if !Context::encode_decode_attr_set(&attrs) {
            attrs.push(RustlerAttr::Encode);
            attrs.push(RustlerAttr::Decode);
        }

        let has_lifetime = match ast.generics.lifetimes().count() {
            0 => false,
            1 => true,
            _ => panic!("Struct can only have one lifetime argument"),
        };

        let ident = &ast.ident;
        let ident_with_lifetime = if has_lifetime {
            quote! { #ident <'a> }
        } else {
            quote! { #ident }
        };

        let variants = match ast.data {
            Data::Enum(ref data_enum) => Some(data_enum.variants.iter().collect()),
            _ => None,
        };

        let struct_fields = match ast.data {
            Data::Struct(ref data_struct) => Some(data_struct.fields.iter().collect()),
            _ => None,
        };

        let is_tuple_struct = match ast.data {
            Data::Struct(ref data_struct) => match data_struct.fields {
                Fields::Unnamed(_) => true,
                _ => false,
            },
            _ => false,
        };

        Self {
            attrs,
            ident,
            ident_with_lifetime,
            variants,
            struct_fields,
            is_tuple_struct,
        }
    }

    pub fn atoms_module_name(&self, span: Span) -> Ident {
        Ident::new(&format!("RUSTLER_ATOMS_{}", self.ident), span)
    }

    pub fn encode(&self) -> bool {
        self.attrs.iter().any(|attr| match attr {
            RustlerAttr::Encode => true,
            _ => false,
        })
    }

    pub fn decode(&self) -> bool {
        self.attrs.iter().any(|attr| match attr {
            RustlerAttr::Decode => true,
            _ => false,
        })
    }

    pub fn field_atoms(&self) -> Option<Vec<TokenStream>> {
        self.struct_fields.as_ref().map(|struct_fields| {
            struct_fields
                .iter()
                .map(|field| {
                    let atom_fun = Self::field_to_atom_fun(field);

                    let ident = field.ident.as_ref().unwrap();
                    let ident_str = ident.to_string();
                    let ident_str = Self::remove_raw(&ident_str);

                    quote! {
                        #atom_fun = #ident_str,
                    }
                })
                .collect()
        })
    }

    pub fn field_to_atom_fun(field: &Field) -> Ident {
        let ident = field.ident.as_ref().unwrap();
        let ident_str = ident.to_string();
        let ident_str = Self::remove_raw(&ident_str);

        Ident::new(&format!("atom_{}", ident_str), Span::call_site())
    }

    pub fn escape_ident_with_index(ident_str: &str, index: usize, infix: &str) -> Ident {
        Ident::new(
            &format!("RUSTLER_{}_field_{}_{}", infix, index, Self::remove_raw(ident_str)),
            Span::call_site()
        )
    }

    pub fn escape_ident(ident_str: &str, infix: &str) -> Ident {
        Ident::new(
            &format!("RUSTLER_{}_field_{}", infix, Self::remove_raw(ident_str)),
            Span::call_site()
        )
    }

    fn remove_raw(ident_str: &str) -> &str {
        ident_str
            .split("r#")
            .last()
            .expect("split has always at least one element")
    }

    fn encode_decode_attr_set(attrs: &[RustlerAttr]) -> bool {
        attrs.iter().any(|attr| match attr {
            RustlerAttr::Encode => true,
            RustlerAttr::Decode => true,
            _ => false,
        })
    }

    fn get_rustler_attrs(attr: &syn::Attribute) -> Vec<RustlerAttr> {
        attr.path
            .segments
            .iter()
            .filter_map(|segment| {
                let meta = attr.parse_meta().expect("can parse meta");
                match segment.ident.to_string().as_ref() {
                    "rustler" => Context::parse_rustler(&meta),
                    "tag" => Context::try_parse_tag(&meta),
                    "module" => Context::try_parse_module(&meta),
                    _ => None,
                }
            })
            .flatten()
            .collect()
    }

    fn parse_rustler(meta: &Meta) -> Option<Vec<RustlerAttr>> {
        if let Meta::List(ref list) = meta {
            return Some(
                list.nested
                    .iter()
                    .map(Context::parse_nested_rustler)
                    .collect(),
            );
        }

        panic!("Expected encode and/or decode in rustler attribute");
    }

    fn parse_nested_rustler(nested: &NestedMeta) -> RustlerAttr {
        if let NestedMeta::Meta(ref meta) = nested {
            if let Meta::Path(ref path) = meta {
                match path.segments[0].ident.to_string().as_ref() {
                    "encode" => return RustlerAttr::Encode,
                    "decode" => return RustlerAttr::Decode,
                    other => panic!("Unexpected literal {}", other),
                }
            }
        }

        panic!("Expected encode and/or decode in rustler attribute");
    }

    fn try_parse_tag(meta: &Meta) -> Option<Vec<RustlerAttr>> {
        if let Meta::NameValue(ref name_value) = meta {
            if let Lit::Str(ref tag) = name_value.lit {
                return Some(vec![RustlerAttr::Tag(tag.value())]);
            }
        }
        panic!("Cannot parse module")
    }

    fn try_parse_module(meta: &Meta) -> Option<Vec<RustlerAttr>> {
        if let Meta::NameValue(name_value) = meta {
            if let Lit::Str(ref module) = name_value.lit {
                let ident = format!("Elixir.{}", module.value());
                return Some(vec![RustlerAttr::Module(ident)]);
            }
        }
        panic!("Cannot parse tag")
    }
}
<commit_msg>Cargo fmt<commit_after>use proc_macro2::{Span, TokenStream};
use syn::{Data, Field, Fields, Ident, Lit, Meta, NestedMeta, Variant};

use super::RustlerAttr;

///
/// A parsing context struct.
///
/// `Context` holds information usable for different codegen modules.
///
pub(crate) struct Context<'a> {
    pub attrs: Vec<RustlerAttr>,
    pub ident: &'a proc_macro2::Ident,
    pub ident_with_lifetime: proc_macro2::TokenStream,
    pub variants: Option<Vec<&'a Variant>>,
    pub struct_fields: Option<Vec<&'a Field>>,
    pub is_tuple_struct: bool,
}

impl<'a> Context<'a> {
    pub fn from_ast(ast: &'a syn::DeriveInput) -> Self {
        let mut attrs: Vec<_> = ast
            .attrs
            .iter()
            .map(Context::get_rustler_attrs)
            .flatten()
            .collect();

        //
        // Default: generate encoder and decoder
        //
        if !Context::encode_decode_attr_set(&attrs) {
            attrs.push(RustlerAttr::Encode);
            attrs.push(RustlerAttr::Decode);
        }

        let has_lifetime = match ast.generics.lifetimes().count() {
            0 => false,
            1 => true,
            _ => panic!("Struct can only have one lifetime argument"),
        };

        let ident = &ast.ident;
        let ident_with_lifetime = if has_lifetime {
            quote! { #ident <'a> }
        } else {
            quote! { #ident }
        };

        let variants = match ast.data {
            Data::Enum(ref data_enum) => Some(data_enum.variants.iter().collect()),
            _ => None,
        };

        let struct_fields = match ast.data {
            Data::Struct(ref data_struct) => Some(data_struct.fields.iter().collect()),
            _ => None,
        };

        let is_tuple_struct = match ast.data {
            Data::Struct(ref data_struct) => match data_struct.fields {
                Fields::Unnamed(_) => true,
                _ => false,
            },
            _ => false,
        };

        Self {
            attrs,
            ident,
            ident_with_lifetime,
            variants,
            struct_fields,
            is_tuple_struct,
        }
    }

    pub fn atoms_module_name(&self, span: Span) -> Ident {
        Ident::new(&format!("RUSTLER_ATOMS_{}", self.ident), span)
    }

    pub fn encode(&self) -> bool {
        self.attrs.iter().any(|attr| match attr {
            RustlerAttr::Encode => true,
            _ => false,
        })
    }

    pub fn decode(&self) -> bool {
        self.attrs.iter().any(|attr| match attr {
            RustlerAttr::Decode => true,
            _ => false,
        })
    }

    pub fn field_atoms(&self) -> Option<Vec<TokenStream>> {
        self.struct_fields.as_ref().map(|struct_fields| {
            struct_fields
                .iter()
                .map(|field| {
                    let atom_fun = Self::field_to_atom_fun(field);

                    let ident = field.ident.as_ref().unwrap();
                    let ident_str = ident.to_string();
                    let ident_str = Self::remove_raw(&ident_str);

                    quote! {
                        #atom_fun = #ident_str,
                    }
                })
                .collect()
        })
    }

    pub fn field_to_atom_fun(field: &Field) -> Ident {
        let ident = field.ident.as_ref().unwrap();
        let ident_str = ident.to_string();
        let ident_str = Self::remove_raw(&ident_str);

        Ident::new(&format!("atom_{}", ident_str), Span::call_site())
    }

    pub fn escape_ident_with_index(ident_str: &str, index: usize, infix: &str) -> Ident {
        Ident::new(
            &format!(
                "RUSTLER_{}_field_{}_{}",
                infix,
                index,
                Self::remove_raw(ident_str)
            ),
            Span::call_site(),
        )
    }

    pub fn escape_ident(ident_str: &str, infix: &str) -> Ident {
        Ident::new(
            &format!("RUSTLER_{}_field_{}", infix, Self::remove_raw(ident_str)),
            Span::call_site(),
        )
    }

    fn remove_raw(ident_str: &str) -> &str {
        ident_str
            .split("r#")
            .last()
            .expect("split has always at least one element")
    }

    fn encode_decode_attr_set(attrs: &[RustlerAttr]) -> bool {
        attrs.iter().any(|attr| match attr {
            RustlerAttr::Encode => true,
            RustlerAttr::Decode => true,
            _ => false,
        })
    }

    fn get_rustler_attrs(attr: &syn::Attribute) -> Vec<RustlerAttr> {
        attr.path
            .segments
            .iter()
            .filter_map(|segment| {
                let meta = attr.parse_meta().expect("can parse meta");
                match segment.ident.to_string().as_ref() {
                    "rustler" => Context::parse_rustler(&meta),
                    "tag" => Context::try_parse_tag(&meta),
                    "module" => Context::try_parse_module(&meta),
                    _ => None,
                }
            })
            .flatten()
            .collect()
    }

    fn parse_rustler(meta: &Meta) -> Option<Vec<RustlerAttr>> {
        if let Meta::List(ref list) = meta {
            return Some(
                list.nested
                    .iter()
                    .map(Context::parse_nested_rustler)
                    .collect(),
            );
        }

        panic!("Expected encode and/or decode in rustler attribute");
    }

    fn parse_nested_rustler(nested: &NestedMeta) -> RustlerAttr {
        if let NestedMeta::Meta(ref meta) = nested {
            if let Meta::Path(ref path) = meta {
                match path.segments[0].ident.to_string().as_ref() {
                    "encode" => return RustlerAttr::Encode,
                    "decode" => return RustlerAttr::Decode,
                    other => panic!("Unexpected literal {}", other),
                }
            }
        }

        panic!("Expected encode and/or decode in rustler attribute");
    }

    fn try_parse_tag(meta: &Meta) -> Option<Vec<RustlerAttr>> {
        if let Meta::NameValue(ref name_value) = meta {
            if let Lit::Str(ref tag) = name_value.lit {
                return Some(vec![RustlerAttr::Tag(tag.value())]);
            }
        }
        panic!("Cannot parse module")
    }

    fn try_parse_module(meta: &Meta) -> Option<Vec<RustlerAttr>> {
        if let Meta::NameValue(name_value) = meta {
            if let Lit::Str(ref module) = name_value.lit {
                let ident = format!("Elixir.{}", module.value());
                return Some(vec![RustlerAttr::Module(ident)]);
            }
        }
        panic!("Cannot parse tag")
    }
}
<|endoftext|>


<commit_before>//! Developer note:
//!
//! This is a set of embed builders for rich embeds.
//!
//! These are used in the [`ChannelId::send_message`] and
//! [`ExecuteWebhook::embeds`] methods, both as part of builders.
//!
//! The only builder that should be exposed is [`CreateEmbed`]. The rest of
//! these have no real reason for being exposed, but are for completeness' sake.
//!
//! Documentation for embeds can be found [here].
//!
//! [`ChannelId::send_message`]: ../model/id/struct.ChannelId.html#method.send_message
//! [`CreateEmbed`]: struct.CreateEmbed.html
//! [`ExecuteWebhook::embeds`]: struct.ExecuteWebhook.html#method.embeds
//! [here]: https://discordapp.com/developers/docs/resources/channel#embed-object

use chrono::{DateTime, TimeZone};
use internal::prelude::*;
use model::channel::Embed;
use serde_json::Value;
use std::{
    default::Default,
    fmt::Display
};
use utils::{self, VecMap};

#[cfg(feature = "utils")]
use utils::Colour;

/// A builder to create a fake [`Embed`] object, for use with the
/// [`ChannelId::send_message`] and [`ExecuteWebhook::embeds`] methods.
///
/// # Examples
///
/// Refer to the documentation for [`ChannelId::send_message`] for a very in-depth
/// example on how to use this.
///
/// [`ChannelId::send_message`]: ../model/id/struct.ChannelId.html#method.send_message
/// [`Embed`]: ../model/channel/struct.Embed.html
/// [`ExecuteWebhook::embeds`]: struct.ExecuteWebhook.html#method.embeds
#[derive(Clone, Debug)]
pub struct CreateEmbed(pub VecMap<&'static str, Value>);

impl CreateEmbed {
    /// Set the author of the embed.
    ///
    /// Refer to the documentation for [`CreateEmbedAuthor`] for more
    /// information.
    ///
    /// [`CreateEmbedAuthor`]: struct.CreateEmbedAuthor.html
    pub fn author<F>(mut self, f: F) -> Self
        where F: FnOnce(CreateEmbedAuthor) -> CreateEmbedAuthor {
        let map = utils::vecmap_to_json_map(f(CreateEmbedAuthor::default()).0);

        self.0.insert("author", Value::Object(map));

        self
    }

    /// Set the colour of the left-hand side of the embed.
    ///
    /// This is an alias of [`colour`].
    ///
    /// [`colour`]: #method.colour
    #[cfg(feature = "utils")]
    #[inline]
    pub fn color<C: Into<Colour>>(self, colour: C) -> Self { self.colour(colour.into()) }

    /// Set the colour of the left-hand side of the embed.
    #[cfg(feature = "utils")]
    pub fn colour<C: Into<Colour>>(mut self, colour: C) -> Self {
        self.0.insert(
            "color",
            Value::Number(Number::from(u64::from(colour.into().0))),
        );

        self
    }

    /// Set the colour of the left-hand side of the embed.
    ///
    /// This is an alias of [`colour`].
    ///
    /// [`colour`]: #method.colour
    #[cfg(not(feature = "utils"))]
    #[inline]
    pub fn color(self, colour: u32) -> Self { self.colour(colour) }

    /// Set the colour of the left-hand side of the embed.
    #[cfg(not(feature = "utils"))]
    pub fn colour(mut self, colour: u32) -> Self {
        self.0
            .insert("color", Value::Number(Number::from(colour)));

        self
    }

    /// Set the description of the embed.
    ///
    /// **Note**: This can't be longer than 2048 characters.
    pub fn description<D: Display>(mut self, description: D) -> Self {
        self.0.insert(
            "description",
            Value::String(description.to_string()),
        );

        self
    }

    /// Set a field. Note that this will not overwrite other fields, and will
    /// add to them.
    ///
    /// **Note**: Maximum amount of characters you can put is 256 in a field
    /// name and 1024 in a field value and a field is inline by default.
    pub fn field<T, U>(mut self, name: T, value: U, inline: bool) -> Self
        where T: Display, U: Display {
        {
            let entry = self.0
                .entry("fields")
                .or_insert_with(|| Value::Array(vec![]));

            if let Value::Array(ref mut inner) = *entry {
                inner.push(json!({
                    "inline": inline,
                    "name": name.to_string(),
                    "value": value.to_string(),
                }));
            }
        }

        self
    }

    /// Adds multiple fields at once.
    ///
    /// This is sugar to reduce the need of calling [`field`] manually multiple times.
    ///
    /// [`field`]: #method.field
    pub fn fields<T, U, It>(mut self, fields: It) -> Self
        where It: IntoIterator<Item=(T, U, bool)>,
              T: Display,
              U: Display {
        for field in fields {
            self = self.field(field.0.to_string(), field.1.to_string(), field.2);
        }

        self
    }

    /// Set the footer of the embed.
    ///
    /// Refer to the documentation for [`CreateEmbedFooter`] for more
    /// information.
    ///
    /// [`CreateEmbedFooter`]: struct.CreateEmbedFooter.html
    pub fn footer<F>(mut self, f: F) -> Self
        where F: FnOnce(CreateEmbedFooter) -> CreateEmbedFooter {
        let footer = f(CreateEmbedFooter::default()).0;
        let map = utils::vecmap_to_json_map(footer);

        self.0.insert("footer", Value::Object(map));

        self
    }

    fn url_object(mut self, name: &'static str, url: &str) -> Self {
        let obj = json!({
            "url": url.to_string()
        });

        self.0.insert(name, obj);

        self
    }

    /// Set the image associated with the embed. This only supports HTTP(S).
    #[inline]
    pub fn image<S: AsRef<str>>(self, url: S) -> Self {
        self.url_object("image", url.as_ref())
    }

    /// Set the thumbnail of the embed. This only supports HTTP(S).
    #[inline]
    pub fn thumbnail<S: AsRef<str>>(self, url: S) -> Self {
        self.url_object("thumbnail", url.as_ref())
    }

    /// Set the timestamp.
    ///
    /// You may pass a direct string:
    ///
    /// - `2017-01-03T23:00:00`
    /// - `2004-06-08T16:04:23`
    /// - `2004-06-08T16:04:23`
    ///
    /// This timestamp must be in ISO-8601 format. It must also be in UTC format.
    ///
    /// You can also pass anything that implements `chrono::TimeZone`.
    ///
    /// # Examples
    ///
    /// Passing a string timestamp:
    ///
    /// ```rust,no_run
    /// use serenity::prelude::*;
    /// use serenity::model::channel::Message;
    ///
    /// struct Handler;
    ///
    /// impl EventHandler for Handler {
    ///     fn message(&self, _: Context, msg: Message) {
    ///         if msg.content == "~embed" {
    ///             let _ = msg.channel_id.send_message(|m| m
    ///              .embed(|e| e
    ///                     .title("hello")
    ///                     .timestamp("2004-06-08T16:04:23")));
    ///         }
    ///     }
    /// }
    ///
    /// let mut client = Client::new("token", Handler).unwrap();
    ///
    /// client.start().unwrap();
    /// ```
    ///
    /// Creating a join-log:
    ///
    /// Note: this example isn't efficient and is for demonstrative purposes.
    ///
    /// ```rust,no_run
    /// use serenity::prelude::*;
    /// use serenity::model::guild::Member;
    /// use serenity::model::id::GuildId;
    ///
    /// struct Handler;
    ///
    /// impl EventHandler for Handler {
    ///     fn guild_member_addition(&self, _: Context, guild_id: GuildId, member: Member) {
    ///         use serenity::CACHE;
    ///         let cache = CACHE.read();
    ///
    ///         if let Some(guild) = cache.guild(guild_id) {
    ///             let guild = guild.read();
    ///
    ///             let channel_search = guild
    ///                 .channels
    ///                 .values()
    ///                 .find(|c| c.read().name == "join-log");
    ///
    ///             if let Some(channel) = channel_search {
    ///                 let user = member.user.read();
    ///
    ///                 let _ = channel.read().send_message(|m| m
    ///                     .embed(|e| {
    ///                         let mut e = e
    ///                             .author(|a| a.icon_url(&user.face()).name(&user.name))
    ///                             .title("Member Join");
    ///
    ///                         if let Some(ref joined_at) = member.joined_at {
    ///                             e = e.timestamp(joined_at);
    ///                         }
    ///
    ///                         e
    ///                     }));
    ///             }
    ///         }
    ///     }
    /// }
    ///
    /// let mut client = Client::new("token", Handler).unwrap();
    ///
    /// client.start().unwrap();
    /// ```
    pub fn timestamp<T: Into<Timestamp>>(mut self, timestamp: T) -> Self {
        self.0
            .insert("timestamp", Value::String(timestamp.into().ts));

        self
    }

    /// Set the title of the embed.
    pub fn title<D: Display>(mut self, title: D) -> Self {
        self.0
            .insert("title", Value::String(title.to_string()));

        self
    }

    /// Set the URL to direct to when clicking on the title.
    pub fn url<S: AsRef<str>>(mut self, url: S) -> Self {
        self.0
            .insert("url", Value::String(url.as_ref().to_string()));

        self
    }

    /// Same as calling [`image`] with "attachment://filename.(jpg, png)".
    ///
    /// Note however, you have to be sure you set an attachment (with [`ChannelId::send_files`])
    /// with the provided filename. Or else this won't work.
    ///
    /// [`ChannelId::send_files`]: ../model/id/struct.ChannelId.html#send_files
    pub fn attachment<S: AsRef<str>>(self, filename: S) -> Self {
        self.image(&format!("attachment://{}", filename.as_ref()))
    }
}

impl Default for CreateEmbed {
    /// Creates a builder with default values, setting the `type` to `rich`.
    fn default() -> CreateEmbed {
        let mut map = VecMap::new();
        map.insert("type", Value::String("rich".to_string()));

        CreateEmbed(map)
    }
}

impl From<Embed> for CreateEmbed {
    /// Converts the fields of an embed into the values for a new embed builder.
    ///
    /// Some values - such as Proxy URLs - are not preserved.
    fn from(embed: Embed) -> CreateEmbed {
        let mut b = CreateEmbed::default().colour(embed.colour);

        if let Some(author) = embed.author {
            b = b.author(move |mut a| {
                a = a.name(&author.name);

                if let Some(icon_url) = author.icon_url {
                    a = a.icon_url(&icon_url);
                }

                if let Some(url) = author.url {
                    a = a.url(&url);
                }

                a
            });
        }

        if let Some(description) = embed.description {
            b = b.description(&description);
        }

        for field in embed.fields {
            b = b.field(field.name, field.value, field.inline);
        }

        if let Some(image) = embed.image {
            b = b.image(&image.url);
        }

        if let Some(timestamp) = embed.timestamp {
            b = b.timestamp(timestamp);
        }

        if let Some(thumbnail) = embed.thumbnail {
            b = b.thumbnail(&thumbnail.url);
        }

        if let Some(url) = embed.url {
            b = b.url(&url);
        }

        if let Some(title) = embed.title {
            b = b.title(&title);
        }

        if let Some(footer) = embed.footer {
            b = b.footer(move |mut f| {
                f = f.text(&footer.text);

                if let Some(icon_url) = footer.icon_url {
                    f = f.icon_url(&icon_url);
                }

                f
            });
        }

        b
    }
}

/// A builder to create a fake [`Embed`] object's author, for use with the
/// [`CreateEmbed::author`] method.
///
/// Requires that you specify a [`name`].
///
/// [`Embed`]: ../model/channel/struct.Embed.html
/// [`CreateEmbed::author`]: struct.CreateEmbed.html#method.author
/// [`name`]: #method.name
#[derive(Clone, Debug, Default)]
pub struct CreateEmbedAuthor(pub VecMap<&'static str, Value>);

impl CreateEmbedAuthor {
    /// Set the URL of the author's icon.
    pub fn icon_url(mut self, icon_url: &str) -> Self {
        self.0.insert("icon_url", Value::String(icon_url.to_string()));

        self
    }

    /// Set the author's name.
    pub fn name(mut self, name: &str) -> Self {
        self.0.insert("name", Value::String(name.to_string()));

        self
    }

    /// Set the author's URL.
    pub fn url(mut self, url: &str) -> Self {
        self.0.insert("url", Value::String(url.to_string()));

        self
    }
}

/// A builder to create a fake [`Embed`] object's footer, for use with the
/// [`CreateEmbed::footer`] method.
///
/// This does not require any field be set.
///
/// [`Embed`]: ../model/channel/struct.Embed.html
/// [`CreateEmbed::footer`]: struct.CreateEmbed.html#method.footer
#[derive(Clone, Debug, Default)]
pub struct CreateEmbedFooter(pub VecMap<&'static str, Value>);

impl CreateEmbedFooter {
    /// Set the icon URL's value. This only supports HTTP(S).
    pub fn icon_url(mut self, icon_url: &str) -> Self {
        self.0.insert("icon_url", Value::String(icon_url.to_string()));

        self
    }

    /// Set the footer's text.
    pub fn text<D: Display>(mut self, text: D) -> Self {
        self.0.insert("text", Value::String(text.to_string()));

        self
    }
}

#[derive(Clone, Debug)]
pub struct Timestamp {
    pub ts: String,
}

impl From<String> for Timestamp {
    fn from(ts: String) -> Self {
        Timestamp {
            ts,
        }
    }
}

impl<'a> From<&'a str> for Timestamp {
    fn from(ts: &'a str) -> Self {
        Timestamp {
            ts: ts.to_string(),
        }
    }
}

impl<'a, Tz: TimeZone> From<&'a DateTime<Tz>> for Timestamp
    where Tz::Offset: Display {
    fn from(dt: &'a DateTime<Tz>) -> Self {
        Timestamp {
            ts: dt.to_rfc3339(),
        }
    }
}
<commit_msg>Fix the notice of `field`<commit_after>//! Developer note:
//!
//! This is a set of embed builders for rich embeds.
//!
//! These are used in the [`ChannelId::send_message`] and
//! [`ExecuteWebhook::embeds`] methods, both as part of builders.
//!
//! The only builder that should be exposed is [`CreateEmbed`]. The rest of
//! these have no real reason for being exposed, but are for completeness' sake.
//!
//! Documentation for embeds can be found [here].
//!
//! [`ChannelId::send_message`]: ../model/id/struct.ChannelId.html#method.send_message
//! [`CreateEmbed`]: struct.CreateEmbed.html
//! [`ExecuteWebhook::embeds`]: struct.ExecuteWebhook.html#method.embeds
//! [here]: https://discordapp.com/developers/docs/resources/channel#embed-object

use chrono::{DateTime, TimeZone};
use internal::prelude::*;
use model::channel::Embed;
use serde_json::Value;
use std::{
    default::Default,
    fmt::Display
};
use utils::{self, VecMap};

#[cfg(feature = "utils")]
use utils::Colour;

/// A builder to create a fake [`Embed`] object, for use with the
/// [`ChannelId::send_message`] and [`ExecuteWebhook::embeds`] methods.
///
/// # Examples
///
/// Refer to the documentation for [`ChannelId::send_message`] for a very in-depth
/// example on how to use this.
///
/// [`ChannelId::send_message`]: ../model/id/struct.ChannelId.html#method.send_message
/// [`Embed`]: ../model/channel/struct.Embed.html
/// [`ExecuteWebhook::embeds`]: struct.ExecuteWebhook.html#method.embeds
#[derive(Clone, Debug)]
pub struct CreateEmbed(pub VecMap<&'static str, Value>);

impl CreateEmbed {
    /// Set the author of the embed.
    ///
    /// Refer to the documentation for [`CreateEmbedAuthor`] for more
    /// information.
    ///
    /// [`CreateEmbedAuthor`]: struct.CreateEmbedAuthor.html
    pub fn author<F>(mut self, f: F) -> Self
        where F: FnOnce(CreateEmbedAuthor) -> CreateEmbedAuthor {
        let map = utils::vecmap_to_json_map(f(CreateEmbedAuthor::default()).0);

        self.0.insert("author", Value::Object(map));

        self
    }

    /// Set the colour of the left-hand side of the embed.
    ///
    /// This is an alias of [`colour`].
    ///
    /// [`colour`]: #method.colour
    #[cfg(feature = "utils")]
    #[inline]
    pub fn color<C: Into<Colour>>(self, colour: C) -> Self { self.colour(colour.into()) }

    /// Set the colour of the left-hand side of the embed.
    #[cfg(feature = "utils")]
    pub fn colour<C: Into<Colour>>(mut self, colour: C) -> Self {
        self.0.insert(
            "color",
            Value::Number(Number::from(u64::from(colour.into().0))),
        );

        self
    }

    /// Set the colour of the left-hand side of the embed.
    ///
    /// This is an alias of [`colour`].
    ///
    /// [`colour`]: #method.colour
    #[cfg(not(feature = "utils"))]
    #[inline]
    pub fn color(self, colour: u32) -> Self { self.colour(colour) }

    /// Set the colour of the left-hand side of the embed.
    #[cfg(not(feature = "utils"))]
    pub fn colour(mut self, colour: u32) -> Self {
        self.0
            .insert("color", Value::Number(Number::from(colour)));

        self
    }

    /// Set the description of the embed.
    ///
    /// **Note**: This can't be longer than 2048 characters.
    pub fn description<D: Display>(mut self, description: D) -> Self {
        self.0.insert(
            "description",
            Value::String(description.to_string()),
        );

        self
    }

    /// Set a field. Note that this will not overwrite other fields, and will
    /// add to them.
    ///
    /// **Note**: Maximum amount of characters you can put is 256 in a field
    /// name and 1024 in a field value.
    pub fn field<T, U>(mut self, name: T, value: U, inline: bool) -> Self
        where T: Display, U: Display {
        {
            let entry = self.0
                .entry("fields")
                .or_insert_with(|| Value::Array(vec![]));

            if let Value::Array(ref mut inner) = *entry {
                inner.push(json!({
                    "inline": inline,
                    "name": name.to_string(),
                    "value": value.to_string(),
                }));
            }
        }

        self
    }

    /// Adds multiple fields at once.
    ///
    /// This is sugar to reduce the need of calling [`field`] manually multiple times.
    ///
    /// [`field`]: #method.field
    pub fn fields<T, U, It>(mut self, fields: It) -> Self
        where It: IntoIterator<Item=(T, U, bool)>,
              T: Display,
              U: Display {
        for field in fields {
            self = self.field(field.0.to_string(), field.1.to_string(), field.2);
        }

        self
    }

    /// Set the footer of the embed.
    ///
    /// Refer to the documentation for [`CreateEmbedFooter`] for more
    /// information.
    ///
    /// [`CreateEmbedFooter`]: struct.CreateEmbedFooter.html
    pub fn footer<F>(mut self, f: F) -> Self
        where F: FnOnce(CreateEmbedFooter) -> CreateEmbedFooter {
        let footer = f(CreateEmbedFooter::default()).0;
        let map = utils::vecmap_to_json_map(footer);

        self.0.insert("footer", Value::Object(map));

        self
    }

    fn url_object(mut self, name: &'static str, url: &str) -> Self {
        let obj = json!({
            "url": url.to_string()
        });

        self.0.insert(name, obj);

        self
    }

    /// Set the image associated with the embed. This only supports HTTP(S).
    #[inline]
    pub fn image<S: AsRef<str>>(self, url: S) -> Self {
        self.url_object("image", url.as_ref())
    }

    /// Set the thumbnail of the embed. This only supports HTTP(S).
    #[inline]
    pub fn thumbnail<S: AsRef<str>>(self, url: S) -> Self {
        self.url_object("thumbnail", url.as_ref())
    }

    /// Set the timestamp.
    ///
    /// You may pass a direct string:
    ///
    /// - `2017-01-03T23:00:00`
    /// - `2004-06-08T16:04:23`
    /// - `2004-06-08T16:04:23`
    ///
    /// This timestamp must be in ISO-8601 format. It must also be in UTC format.
    ///
    /// You can also pass anything that implements `chrono::TimeZone`.
    ///
    /// # Examples
    ///
    /// Passing a string timestamp:
    ///
    /// ```rust,no_run
    /// use serenity::prelude::*;
    /// use serenity::model::channel::Message;
    ///
    /// struct Handler;
    ///
    /// impl EventHandler for Handler {
    ///     fn message(&self, _: Context, msg: Message) {
    ///         if msg.content == "~embed" {
    ///             let _ = msg.channel_id.send_message(|m| m
    ///              .embed(|e| e
    ///                     .title("hello")
    ///                     .timestamp("2004-06-08T16:04:23")));
    ///         }
    ///     }
    /// }
    ///
    /// let mut client = Client::new("token", Handler).unwrap();
    ///
    /// client.start().unwrap();
    /// ```
    ///
    /// Creating a join-log:
    ///
    /// Note: this example isn't efficient and is for demonstrative purposes.
    ///
    /// ```rust,no_run
    /// use serenity::prelude::*;
    /// use serenity::model::guild::Member;
    /// use serenity::model::id::GuildId;
    ///
    /// struct Handler;
    ///
    /// impl EventHandler for Handler {
    ///     fn guild_member_addition(&self, _: Context, guild_id: GuildId, member: Member) {
    ///         use serenity::CACHE;
    ///         let cache = CACHE.read();
    ///
    ///         if let Some(guild) = cache.guild(guild_id) {
    ///             let guild = guild.read();
    ///
    ///             let channel_search = guild
    ///                 .channels
    ///                 .values()
    ///                 .find(|c| c.read().name == "join-log");
    ///
    ///             if let Some(channel) = channel_search {
    ///                 let user = member.user.read();
    ///
    ///                 let _ = channel.read().send_message(|m| m
    ///                     .embed(|e| {
    ///                         let mut e = e
    ///                             .author(|a| a.icon_url(&user.face()).name(&user.name))
    ///                             .title("Member Join");
    ///
    ///                         if let Some(ref joined_at) = member.joined_at {
    ///                             e = e.timestamp(joined_at);
    ///                         }
    ///
    ///                         e
    ///                     }));
    ///             }
    ///         }
    ///     }
    /// }
    ///
    /// let mut client = Client::new("token", Handler).unwrap();
    ///
    /// client.start().unwrap();
    /// ```
    pub fn timestamp<T: Into<Timestamp>>(mut self, timestamp: T) -> Self {
        self.0
            .insert("timestamp", Value::String(timestamp.into().ts));

        self
    }

    /// Set the title of the embed.
    pub fn title<D: Display>(mut self, title: D) -> Self {
        self.0
            .insert("title", Value::String(title.to_string()));

        self
    }

    /// Set the URL to direct to when clicking on the title.
    pub fn url<S: AsRef<str>>(mut self, url: S) -> Self {
        self.0
            .insert("url", Value::String(url.as_ref().to_string()));

        self
    }

    /// Same as calling [`image`] with "attachment://filename.(jpg, png)".
    ///
    /// Note however, you have to be sure you set an attachment (with [`ChannelId::send_files`])
    /// with the provided filename. Or else this won't work.
    ///
    /// [`ChannelId::send_files`]: ../model/id/struct.ChannelId.html#send_files
    pub fn attachment<S: AsRef<str>>(self, filename: S) -> Self {
        self.image(&format!("attachment://{}", filename.as_ref()))
    }
}

impl Default for CreateEmbed {
    /// Creates a builder with default values, setting the `type` to `rich`.
    fn default() -> CreateEmbed {
        let mut map = VecMap::new();
        map.insert("type", Value::String("rich".to_string()));

        CreateEmbed(map)
    }
}

impl From<Embed> for CreateEmbed {
    /// Converts the fields of an embed into the values for a new embed builder.
    ///
    /// Some values - such as Proxy URLs - are not preserved.
    fn from(embed: Embed) -> CreateEmbed {
        let mut b = CreateEmbed::default().colour(embed.colour);

        if let Some(author) = embed.author {
            b = b.author(move |mut a| {
                a = a.name(&author.name);

                if let Some(icon_url) = author.icon_url {
                    a = a.icon_url(&icon_url);
                }

                if let Some(url) = author.url {
                    a = a.url(&url);
                }

                a
            });
        }

        if let Some(description) = embed.description {
            b = b.description(&description);
        }

        for field in embed.fields {
            b = b.field(field.name, field.value, field.inline);
        }

        if let Some(image) = embed.image {
            b = b.image(&image.url);
        }

        if let Some(timestamp) = embed.timestamp {
            b = b.timestamp(timestamp);
        }

        if let Some(thumbnail) = embed.thumbnail {
            b = b.thumbnail(&thumbnail.url);
        }

        if let Some(url) = embed.url {
            b = b.url(&url);
        }

        if let Some(title) = embed.title {
            b = b.title(&title);
        }

        if let Some(footer) = embed.footer {
            b = b.footer(move |mut f| {
                f = f.text(&footer.text);

                if let Some(icon_url) = footer.icon_url {
                    f = f.icon_url(&icon_url);
                }

                f
            });
        }

        b
    }
}

/// A builder to create a fake [`Embed`] object's author, for use with the
/// [`CreateEmbed::author`] method.
///
/// Requires that you specify a [`name`].
///
/// [`Embed`]: ../model/channel/struct.Embed.html
/// [`CreateEmbed::author`]: struct.CreateEmbed.html#method.author
/// [`name`]: #method.name
#[derive(Clone, Debug, Default)]
pub struct CreateEmbedAuthor(pub VecMap<&'static str, Value>);

impl CreateEmbedAuthor {
    /// Set the URL of the author's icon.
    pub fn icon_url(mut self, icon_url: &str) -> Self {
        self.0.insert("icon_url", Value::String(icon_url.to_string()));

        self
    }

    /// Set the author's name.
    pub fn name(mut self, name: &str) -> Self {
        self.0.insert("name", Value::String(name.to_string()));

        self
    }

    /// Set the author's URL.
    pub fn url(mut self, url: &str) -> Self {
        self.0.insert("url", Value::String(url.to_string()));

        self
    }
}

/// A builder to create a fake [`Embed`] object's footer, for use with the
/// [`CreateEmbed::footer`] method.
///
/// This does not require any field be set.
///
/// [`Embed`]: ../model/channel/struct.Embed.html
/// [`CreateEmbed::footer`]: struct.CreateEmbed.html#method.footer
#[derive(Clone, Debug, Default)]
pub struct CreateEmbedFooter(pub VecMap<&'static str, Value>);

impl CreateEmbedFooter {
    /// Set the icon URL's value. This only supports HTTP(S).
    pub fn icon_url(mut self, icon_url: &str) -> Self {
        self.0.insert("icon_url", Value::String(icon_url.to_string()));

        self
    }

    /// Set the footer's text.
    pub fn text<D: Display>(mut self, text: D) -> Self {
        self.0.insert("text", Value::String(text.to_string()));

        self
    }
}

#[derive(Clone, Debug)]
pub struct Timestamp {
    pub ts: String,
}

impl From<String> for Timestamp {
    fn from(ts: String) -> Self {
        Timestamp {
            ts,
        }
    }
}

impl<'a> From<&'a str> for Timestamp {
    fn from(ts: &'a str) -> Self {
        Timestamp {
            ts: ts.to_string(),
        }
    }
}

impl<'a, Tz: TimeZone> From<&'a DateTime<Tz>> for Timestamp
    where Tz::Offset: Display {
    fn from(dt: &'a DateTime<Tz>) -> Self {
        Timestamp {
            ts: dt.to_rfc3339(),
        }
    }
}
<|endoftext|>


<commit_before>//! [POST /_matrix/federation/v1/user/keys/query](https://matrix.org/docs/spec/server_server/r0.1.4#post-matrix-federation-v1-user-keys-claim)

use std::collections::BTreeMap;

use ruma_api::ruma_api;
#[cfg(feature = "unstable-pre-spec")]
use ruma_common::encryption::CrossSigningKey;
use ruma_common::encryption::DeviceKeys;
use ruma_identifiers::{DeviceIdBox, UserId};

ruma_api! {
    metadata: {
        description: "Returns the current devices and identity keys for the given users.",
        method: POST,
        name: "get_keys",
        path: "/_matrix/federation/v1/user/keys/query",
        rate_limited: false,
        authentication: ServerSignatures,
    }

    request: {
        /// The keys to be downloaded. Gives all keys for a given user if the list of device ids is
        /// empty.
        pub device_keys: BTreeMap<UserId, Vec<DeviceIdBox>>,
    }

    #[derive(Default)]
    response: {
        /// Keys from the queried devices.
        pub device_keys: BTreeMap<UserId, BTreeMap<DeviceIdBox, DeviceKeys>>,

        /// Information on the master cross-signing keys of the queried users.
        #[cfg(feature = "unstable-pre-spec")]
        #[cfg_attr(docsrs, doc(cfg(feature = "unstable-pre-spec")))]
        #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
        pub master_keys: BTreeMap<UserId, CrossSigningKey>,

        /// Information on the self-signing keys of the queried users.
        #[cfg(feature = "unstable-pre-spec")]
        #[cfg_attr(docsrs, doc(cfg(feature = "unstable-pre-spec")))]
        #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
        pub self_signing_keys: BTreeMap<UserId, CrossSigningKey>,
    }
}

impl Request {
    /// Creates a new `Request` asking for the given device keys.
    pub fn new(device_keys: BTreeMap<UserId, Vec<DeviceIdBox>>) -> Self {
        Self { device_keys }
    }
}

impl Response {
    /// Creates a new `Response` with the given device keys.
    pub fn new(device_keys: BTreeMap<UserId, BTreeMap<DeviceIdBox, DeviceKeys>>) -> Self {
        Self { device_keys, ..Default::default() }
    }
}
<commit_msg>Fix link in doc comment in ruma-federation-api::keys::get_keys::v1<commit_after>//! [POST /_matrix/federation/v1/user/keys/query](https://matrix.org/docs/spec/server_server/r0.1.4#post-matrix-federation-v1-user-keys-query)

use std::collections::BTreeMap;

use ruma_api::ruma_api;
#[cfg(feature = "unstable-pre-spec")]
use ruma_common::encryption::CrossSigningKey;
use ruma_common::encryption::DeviceKeys;
use ruma_identifiers::{DeviceIdBox, UserId};

ruma_api! {
    metadata: {
        description: "Returns the current devices and identity keys for the given users.",
        method: POST,
        name: "get_keys",
        path: "/_matrix/federation/v1/user/keys/query",
        rate_limited: false,
        authentication: ServerSignatures,
    }

    request: {
        /// The keys to be downloaded. Gives all keys for a given user if the list of device ids is
        /// empty.
        pub device_keys: BTreeMap<UserId, Vec<DeviceIdBox>>,
    }

    #[derive(Default)]
    response: {
        /// Keys from the queried devices.
        pub device_keys: BTreeMap<UserId, BTreeMap<DeviceIdBox, DeviceKeys>>,

        /// Information on the master cross-signing keys of the queried users.
        #[cfg(feature = "unstable-pre-spec")]
        #[cfg_attr(docsrs, doc(cfg(feature = "unstable-pre-spec")))]
        #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
        pub master_keys: BTreeMap<UserId, CrossSigningKey>,

        /// Information on the self-signing keys of the queried users.
        #[cfg(feature = "unstable-pre-spec")]
        #[cfg_attr(docsrs, doc(cfg(feature = "unstable-pre-spec")))]
        #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
        pub self_signing_keys: BTreeMap<UserId, CrossSigningKey>,
    }
}

impl Request {
    /// Creates a new `Request` asking for the given device keys.
    pub fn new(device_keys: BTreeMap<UserId, Vec<DeviceIdBox>>) -> Self {
        Self { device_keys }
    }
}

impl Response {
    /// Creates a new `Response` with the given device keys.
    pub fn new(device_keys: BTreeMap<UserId, BTreeMap<DeviceIdBox, DeviceKeys>>) -> Self {
        Self { device_keys, ..Default::default() }
    }
}
<|endoftext|>


<commit_before>#[macro_use(s)]
extern crate ndarray;
extern crate autograd as ag;

use std::time::Instant;

// This is softmax regression with Adam optimizer for mnist.
// 0.918 test accuracy after 3 epochs,
// 0.26 sec/epoch on 2.7GHz Intel Core i5 (blas feature is disabled)
//
// First, run "./download_mnist.sh" beforehand if you don't have dataset and then run
// "cargo run --example mlp_mnist --release" in `examples` dicectory.

macro_rules! eval_with_time {
  ($x:expr) => {
    {
      let start = Instant::now();
      let result = $x;
      let end = start.elapsed();
      println!("{}.{:03} sec", end.as_secs(), end.subsec_nanos() / 1_000_000);
      result
    }
  };
}

fn main()
{
    let ((x_train, y_train), (x_test, y_test)) = dataset::load();

    // -- graph def --
    let ref x = ag::placeholder(&[-1, 28 * 28]);
    let ref y = ag::placeholder(&[-1, 1]);
    let ref w = ag::variable(ag::ndarray_ext::glorot_uniform(&[28 * 28, 10]));
    let ref b = ag::variable(ag::ndarray_ext::zeros(&[1, 10]));
    let ref z = ag::matmul(x, w) + b;
    let ref loss = ag::sparse_softmax_cross_entropy(z, y);
    let ref grads = ag::grad(&[loss], &[w, b]);
    let ref predictions = ag::argmax(z, -1, true);
    let ref accuracy = ag::reduce_mean(&ag::equal(predictions, y), &[0], false);

    // -- actual training --
    let mut optimizer = ag::gradient_descent::Adam { ..Default::default() };
    let max_epoch = 3;
    let batch_size = 200isize;
    let num_samples = x_train.shape()[0];
    let num_batches = num_samples / batch_size as usize;

    for epoch in 0..max_epoch {
        eval_with_time!({
            let perm = ag::ndarray_ext::permutation(num_batches) * batch_size as usize;
            for i in perm.to_vec().into_iter() {
                let i = i as isize;
                let x_batch = x_train.slice(s![i..i + batch_size, ..]).to_owned();
                let y_batch = y_train.slice(s![i..i + batch_size, ..]).to_owned();
                ag::gradient_descent::update(
                    &[w, b],
                    grads,
                    &mut optimizer,
                    &[(x, &x_batch), (y, &y_batch)],
                );
            }
        });
        println!("finish epoch {}", epoch);
    }

    // -- test --
    println!(
        "test accuracy: {}",
        accuracy.eval(&[(x, &x_test), (y, &y_test)])
    );
}

pub mod dataset {
    extern crate ndarray;
    use std::fs::File;
    use std::io;
    use std::io::Read;
    use std::mem;
    use std::path::Path;

    type NdArray = ndarray::Array<f32, ndarray::IxDyn>;

    /// load mnist dataset as "ndarray" objects.
    ///
    /// labels are sparse (vertical vector).
    pub fn load() -> ((NdArray, NdArray), (NdArray, NdArray))
    {
        // load dataset as `Vec`s
        let (train_x, num_image_train): (Vec<f32>, usize) =
            load_images("data/mnist/train-images-idx3-ubyte");
        let (train_y, num_label_train): (Vec<f32>, usize) =
            load_labels("data/mnist/train-labels-idx1-ubyte");
        let (test_x, num_image_test): (Vec<f32>, usize) =
            load_images("data/mnist/t10k-images-idx3-ubyte");
        let (test_y, num_label_test): (Vec<f32>, usize) =
            load_labels("data/mnist/t10k-labels-idx1-ubyte");

        // Vec to ndarray
        let as_arr = NdArray::from_shape_vec;
        let x_train = as_arr(ndarray::IxDyn(&[num_image_train, 28 * 28]), train_x).unwrap();
        let y_train = as_arr(ndarray::IxDyn(&[num_label_train, 1]), train_y).unwrap();
        let x_test = as_arr(ndarray::IxDyn(&[num_image_test, 28 * 28]), test_x).unwrap();
        let y_test = as_arr(ndarray::IxDyn(&[num_label_test, 1]), test_y).unwrap();
        ((x_train, y_train), (x_test, y_test))
    }

    fn load_images<P: AsRef<Path>>(path: P) -> (Vec<f32>, usize)
    {
        let ref mut buf_reader = io::BufReader::new(File::open(path).expect(
            "Please run ./download_mnist.sh beforehand",
        ));
        let magic = u32::from_be(read_u32(buf_reader));
        if magic != 2051 {
            panic!("Invalid magic number. expected 2051, got {}", magic)
        }
        let num_image = u32::from_be(read_u32(buf_reader)) as usize;
        let rows = u32::from_be(read_u32(buf_reader)) as usize;
        let cols = u32::from_be(read_u32(buf_reader)) as usize;
        assert!(rows == 28 && cols == 28);

        // read images
        let mut buf: Vec<u8> = vec![0u8; num_image * rows * cols];
        let _ = buf_reader.read_exact(buf.as_mut());
        let ret = buf.into_iter().map(|x| (x as f32) / 255.).collect();
        (ret, num_image)
    }

    fn load_labels<P: AsRef<Path>>(path: P) -> (Vec<f32>, usize)
    {
        let ref mut buf_reader = io::BufReader::new(File::open(path).unwrap());
        let magic = u32::from_be(read_u32(buf_reader));
        if magic != 2049 {
            panic!("Invalid magic number. Got expect 2049, got {}", magic);
        }
        let num_label = u32::from_be(read_u32(buf_reader)) as usize;
        // read labels
        let mut buf: Vec<u8> = vec![0u8; num_label];
        let _ = buf_reader.read_exact(buf.as_mut());
        let ret: Vec<f32> = buf.into_iter().map(|x| x as f32).collect();
        (ret, num_label)
    }

    fn read_u32<T: Read>(reader: &mut T) -> u32
    {
        let mut buf: [u8; 4] = [0, 0, 0, 0];
        let _ = reader.read_exact(&mut buf);
        unsafe { mem::transmute(buf) }
    }
}
<commit_msg>Style fix<commit_after>#[macro_use(s)]
extern crate ndarray;
extern crate autograd as ag;

use std::time::Instant;

// This is softmax regression with Adam optimizer for mnist.
// 0.918 test accuracy after 3 epochs,
// 0.26 sec/epoch on 2.7GHz Intel Core i5 (blas feature is disabled)
//
// First, run "./download_mnist.sh" beforehand if you don't have dataset and then run
// "cargo run --example mlp_mnist --release" in `examples` dicectory.

macro_rules! eval_with_time {
  ($x:expr) => {
    {
      let start = Instant::now();
      let result = $x;
      let end = start.elapsed();
      println!("{}.{:03} sec", end.as_secs(), end.subsec_nanos() / 1_000_000);
      result
    }
  };
}

fn main()
{
    let ((x_train, y_train), (x_test, y_test)) = dataset::load();

    // -- graph def --
    let ref x = ag::placeholder(&[-1, 28 * 28]);
    let ref y = ag::placeholder(&[-1, 1]);
    let ref w = ag::variable(ag::ndarray_ext::glorot_uniform(&[28 * 28, 10]));
    let ref b = ag::variable(ag::ndarray_ext::zeros(&[1, 10]));
    let ref z = ag::matmul(x, w) + b;
    let ref loss = ag::sparse_softmax_cross_entropy(z, y);
    let ref grads = ag::grad(&[loss], &[w, b]);
    let ref predictions = ag::argmax(z, -1, true);
    let ref accuracy = ag::reduce_mean(&ag::equal(predictions, y), &[0], false);

    // -- actual training --
    let mut optimizer = ag::gradient_descent::Adam { ..Default::default() };
    let max_epoch = 3;
    let batch_size = 200isize;
    let num_samples = x_train.shape()[0];
    let num_batches = num_samples / batch_size as usize;

    for epoch in 0..max_epoch {
        eval_with_time!({
            let mut perm = ag::ndarray_ext::permutation(num_batches);
            perm *= batch_size as usize;
            for i in perm.into_iter() {
                let i = *i as isize;
                let x_batch = x_train.slice(s![i..i + batch_size, ..]).to_owned();
                let y_batch = y_train.slice(s![i..i + batch_size, ..]).to_owned();
                ag::gradient_descent::update(
                    &[w, b],
                    grads,
                    &mut optimizer,
                    &[(x, &x_batch), (y, &y_batch)],
                );
            }
        });
        println!("finish epoch {}", epoch);
    }

    // -- test --
    println!(
        "test accuracy: {}",
        accuracy.eval(&[(x, &x_test), (y, &y_test)])
    );
}

pub mod dataset {
    extern crate ndarray;
    use std::fs::File;
    use std::io;
    use std::io::Read;
    use std::mem;
    use std::path::Path;

    type NdArray = ndarray::Array<f32, ndarray::IxDyn>;

    /// load mnist dataset as "ndarray" objects.
    ///
    /// labels are sparse (vertical vector).
    pub fn load() -> ((NdArray, NdArray), (NdArray, NdArray))
    {
        // load dataset as `Vec`s
        let (train_x, num_image_train): (Vec<f32>, usize) =
            load_images("data/mnist/train-images-idx3-ubyte");
        let (train_y, num_label_train): (Vec<f32>, usize) =
            load_labels("data/mnist/train-labels-idx1-ubyte");
        let (test_x, num_image_test): (Vec<f32>, usize) =
            load_images("data/mnist/t10k-images-idx3-ubyte");
        let (test_y, num_label_test): (Vec<f32>, usize) =
            load_labels("data/mnist/t10k-labels-idx1-ubyte");

        // Vec to ndarray
        let as_arr = NdArray::from_shape_vec;
        let x_train = as_arr(ndarray::IxDyn(&[num_image_train, 28 * 28]), train_x).unwrap();
        let y_train = as_arr(ndarray::IxDyn(&[num_label_train, 1]), train_y).unwrap();
        let x_test = as_arr(ndarray::IxDyn(&[num_image_test, 28 * 28]), test_x).unwrap();
        let y_test = as_arr(ndarray::IxDyn(&[num_label_test, 1]), test_y).unwrap();
        ((x_train, y_train), (x_test, y_test))
    }

    fn load_images<P: AsRef<Path>>(path: P) -> (Vec<f32>, usize)
    {
        let ref mut buf_reader = io::BufReader::new(File::open(path).expect(
            "Please run ./download_mnist.sh beforehand",
        ));
        let magic = u32::from_be(read_u32(buf_reader));
        if magic != 2051 {
            panic!("Invalid magic number. expected 2051, got {}", magic)
        }
        let num_image = u32::from_be(read_u32(buf_reader)) as usize;
        let rows = u32::from_be(read_u32(buf_reader)) as usize;
        let cols = u32::from_be(read_u32(buf_reader)) as usize;
        assert!(rows == 28 && cols == 28);

        // read images
        let mut buf: Vec<u8> = vec![0u8; num_image * rows * cols];
        let _ = buf_reader.read_exact(buf.as_mut());
        let ret = buf.into_iter().map(|x| (x as f32) / 255.).collect();
        (ret, num_image)
    }

    fn load_labels<P: AsRef<Path>>(path: P) -> (Vec<f32>, usize)
    {
        let ref mut buf_reader = io::BufReader::new(File::open(path).unwrap());
        let magic = u32::from_be(read_u32(buf_reader));
        if magic != 2049 {
            panic!("Invalid magic number. Got expect 2049, got {}", magic);
        }
        let num_label = u32::from_be(read_u32(buf_reader)) as usize;
        // read labels
        let mut buf: Vec<u8> = vec![0u8; num_label];
        let _ = buf_reader.read_exact(buf.as_mut());
        let ret: Vec<f32> = buf.into_iter().map(|x| x as f32).collect();
        (ret, num_label)
    }

    fn read_u32<T: Read>(reader: &mut T) -> u32
    {
        let mut buf: [u8; 4] = [0, 0, 0, 0];
        let _ = reader.read_exact(&mut buf);
        unsafe { mem::transmute(buf) }
    }
}
<|endoftext|>


<commit_before>#![feature(core)]
#![feature(str_words)]

extern crate mmap;
extern crate time;
extern crate core;
extern crate timely;
extern crate columnar;
extern crate dataflow_join;

extern crate docopt;
use docopt::Docopt;

use std::io::{BufRead, BufReader, BufWriter, Write, stdin};
use std::fs::File;
use std::rc::Rc;
use std::cell::RefCell;
use std::thread;
use core::raw::Slice as RawSlice;

use dataflow_join::GenericJoinExt;
use dataflow_join::graph::{GraphTrait, GraphVector, GraphMMap, GraphExtenderExt, gallop};

use timely::progress::scope::Scope;
use timely::progress::nested::subgraph::new_graph;
use timely::progress::nested::product::Product;
use timely::example::*;
use timely::communication::*;

use timely::progress::Graph;

static USAGE: &'static str = "
Usage: triangles dataflow (text | binary) <source> <workers> [--inspect] [--interactive]
       triangles compute (text | binary) <source>
       triangles digest <source> <target>
       triangles help
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    if args.get_bool("dataflow") {
        let inspect = args.get_bool("--inspect");
        let interactive = args.get_bool("--interactive");
        let workers = if let Ok(threads) = args.get_str("<workers>").parse() { threads }
                      else { panic!("invalid setting for workers: {}", args.get_str("-t")) };;
        println!("starting triangles dataflow with {:?} worker{}; inspection: {:?}, interactive: {:?}", workers, if workers == 1 { "" } else { "s" }, inspect, interactive);
        let source = args.get_str("<source>");
        if args.get_bool("text") {
            let mut graph = livejournal(&source);
            organize_graph(&mut graph);
            triangles_multi(ProcessCommunicator::new_vector(workers), |index, peers| extract_fragment(&graph, index, peers), inspect, interactive);
        }
        if args.get_bool("binary") {
            triangles_multi(ProcessCommunicator::new_vector(workers), |_, _| GraphMMap::new(&source), inspect, interactive);
        }
    }
    if args.get_bool("compute") {
        let source = args.get_str("<source>");
        if args.get_bool("text") {
            let mut graph = livejournal(&source);
            organize_graph(&mut graph);
            let graph = extract_fragment(&graph, 0, 1);
            println!("triangles: {:?}", raw_triangles(&graph));
        }
        if args.get_bool("binary") {
            let graph = GraphMMap::new(&source);
            println!("triangles: {:?}", raw_triangles(&graph));
        }
    }
    if args.get_bool("digest") {
        println!("digest will overwrite <target>.targets and <target>.offsets, so careful");
        println!("at least, it will once you edit the code to uncomment the line.");
        let source = args.get_str("<source>");
        let _target = args.get_str("<target>");
        let mut graph = livejournal(&source);
        organize_graph(&mut graph);
        // _digest_graph_vector(&extract_fragment(&graph, 0, 1), _target); // will overwrite "prefix.offsets" and "prefix.targets"
    }
    if args.get_bool("help") {
        println!("the code presently assumes you have access to the livejournal graph, from:");
        println!("   https://snap.stanford.edu/data/soc-LiveJournal1.html");
        println!("");
        println!("stash that somewhere, and use it as text source for compute or digest.");
        println!("digest will overwrite <target>.targets and <target>.offsets, so careful");
        println!("at least, it will once you edit the code to uncomment the line.")
    }
}

fn intersect<E: Ord>(aaa: &[E], mut bbb: &[E]) -> u64 {
    let mut count = 0;
    for a in aaa {
        bbb = gallop(bbb, a);
        if bbb.len() > 0 && &bbb[0] == a { count += 1; }
    }
    count
}

fn raw_triangles<G: GraphTrait<Target=u32>>(graph: &G) -> u64 {
    let mut count = 0;
    for a in (0..graph.nodes()) {
        let aaa = graph.edges(a);
        for &b in aaa {
            // println!("considering edge {} {}", a, b);
            let bbb = graph.edges(b as usize);
            count += if aaa.len() < bbb.len() { intersect(aaa.clone(), bbb) }
                     else                     { intersect(bbb, aaa.clone()) };
        }
    }
    count
}

fn triangles_multi<C: Communicator+Send, G: GraphTrait<Target=u32>, F: Fn(u64, u64)->G+Send+Sync>(communicators: Vec<C>, loader: F, inspect: bool, interactive: bool) {
    let mut guards = Vec::new();
    let loader = &loader;
    for communicator in communicators.into_iter() {
        guards.push(thread::Builder::new().name(format!("worker thread {}", communicator.index()))
                                          .scoped(move || triangles(communicator, loader, inspect, interactive))
                                          .unwrap());
    }
}

fn triangles<C: Communicator, G: GraphTrait<Target=u32>, F: Fn(u64, u64)->G>(communicator: C, loader: &F, inspect: bool, interactive: bool) {
    let comm_index = communicator.index();
    let comm_peers = communicator.peers();

    let graph = Rc::new(RefCell::new(loader(comm_index, comm_peers)));
    let mut computation = new_graph(communicator);
    let (mut input, mut stream) = computation.new_input::<u32>();

    // extend u32s to pairs, then pairs to triples.
    let mut triangles = stream.extend(vec![&graph.extend_using(|| { |&a| a as u64 } )])
                              .flat_map(|(p,es)| es.into_iter().map(move |e| (p.clone(), e)))
                              .extend(vec![&graph.extend_using(|| { |&(a,_)| a as u64 }),
                                           &graph.extend_using(|| { |&(_,b)| b as u64 })]);
                            //   .flat_map(|(p,es)| es.into_iter().map(move |e| (p.clone(), e)));

    // let mut triangles = _quads(&mut triangles, &graph);

     if inspect {
         triangles.inspect(|x| println!("triangles: {:?}", x));
     }

    computation.0.borrow_mut().get_internal_summary();
    computation.0.borrow_mut().set_external_summary(Vec::new(), &mut Vec::new());

    if interactive {
        let mut stdinput = stdin();
        let mut line = String::new();
        for index in (0..) {
            stdinput.read_line(&mut line).unwrap();
            let start = time::precise_time_ns();
            if let Some(word) = line.words().next() {
                let read = word.parse();
                if let Ok(number) = read {
                    input.send_messages(&Product::new((), index as u64), vec![number]);
                }
            }
            line.clear();

            input.advance(&Product::new((), index as u64), &Product::new((), index as u64 + 1));
            for _ in (0..5) {
                computation.0.borrow_mut().pull_internal_progress(&mut Vec::new(), &mut Vec::new(), &mut Vec::new());
            }

            println!("elapsed: {:?}us", (time::precise_time_ns() - start)/1000);
        }
    }
    else {
        let step = 1000;
        let nodes = graph.borrow().nodes() - 1;
        let limit = (nodes / step) + 1;
        for index in (0..limit) {
            let index = index as usize;
            let data = ((index * step) as u32 .. ((index + 1) * step) as u32).filter(|&x| x as u64 % comm_peers == comm_index)
                                                                             .filter(|&x| x < nodes as u32)
                                                                             .collect();
            input.send_messages(&Product::new((), index as u64), data);
            input.advance(&Product::new((), index as u64), &Product::new((), index as u64 + 1));
            computation.0.borrow_mut().pull_internal_progress(&mut Vec::new(), &mut Vec::new(), &mut Vec::new());
        }
        input.close_at(&Product::new((), limit as u64));
        while computation.0.borrow_mut().pull_internal_progress(&mut Vec::new(), &mut Vec::new(), &mut Vec::new()) { }
    }
}

fn _quads<G: Graph, G2: GraphTrait<Target=u32>>(stream: &mut Stream<G, ((u32, u32), u32)>, graph: &Rc<RefCell<G2>>) -> Stream<G, (((u32, u32), u32), u32)> {
    //
    stream.extend(vec![&graph.extend_using(|| { |&((a,_),_)| a as u64 }),
                       &graph.extend_using(|| { |&((_,b),_)| b as u64 }),
                       &graph.extend_using(|| { |&((_,_),c)| c as u64 })])
          .flat_map(|(p,es)| es.into_iter().map(move |e| (p.clone(), e)))
    //
    //  let mut fives = quads.extend(vec![Box::new(fragment.extend_using(|| { |&(((a,_),_),_)| a as u64 })),
    //                                    Box::new(fragment.extend_using(|| { |&(((_,b),_),_)| b as u64 })),
    //                                    Box::new(fragment.extend_using(|| { |&(((_,_),c),_)| c as u64 })),
    //                                    Box::new(fragment.extend_using(|| { |&(((_,_),_),d)| d as u64 }))]).flatten();
    //
    // let mut sixes = fives.extend(vec![Box::new(fragment.extend_using(|| { |&((((a,_),_),_),_)| a as u64 })),
    //                                   Box::new(fragment.extend_using(|| { |&((((_,b),_),_),_)| b as u64 })),
    //                                   Box::new(fragment.extend_using(|| { |&((((_,_),c),_),_)| c as u64 })),
    //                                   Box::new(fragment.extend_using(|| { |&((((_,_),_),d),_)| d as u64 })),
    //                                   Box::new(fragment.extend_using(|| { |&((((_,_),_),_),e)| e as u64 }))]).flatten();
    //
    // sixes.observe();
}

fn redirect(edges: &mut Vec<(u32, u32)>) {
    let mut degrees = Vec::new();
    for &(src, dst) in edges.iter() {
        while src as usize >= degrees.len() { degrees.push(0); }
        while dst as usize >= degrees.len() { degrees.push(0); }
        degrees[src as usize] += 1;
        degrees[dst as usize] += 1;
    }

    for index in (0..edges.len()) {
        if degrees[edges[index].0 as usize] > degrees[edges[index].1 as usize] {
            edges[index] = (edges[index].1, edges[index].0);
        }
    }
}

// loads the livejournal file available at https://snap.stanford.edu/data/soc-LiveJournal1.html
fn livejournal(filename: &str) -> Vec<(u32, u32)> {
    let mut graph = Vec::new();
    let file = BufReader::new(File::open(filename).unwrap());
    for readline in file.lines() {
        let line = readline.ok().expect("read error");
        if !line.starts_with('#') {
            let elts: Vec<&str> = line[..].words().collect();
            let src: u32 = elts[0].parse().ok().expect("malformed src");
            let dst: u32 = elts[1].parse().ok().expect("malformed dst");
            if src < dst { graph.push((src, dst)) }
            if src > dst { graph.push((dst, src)) }
        }
    }

    println!("graph data loaded; {:?} edges", graph.len());
    return graph;
}

fn organize_graph(graph: &mut Vec<(u32, u32)>) {

    for index in (0..graph.len()) {
        if graph[index].1 < graph[index].0 {
            graph[index] = (graph[index].1, graph[index].0);
        }
    }

    graph.sort();
    println!("graph data sorted; {:?} edges", graph.len());

    graph.dedup();
    println!("graph data uniqed; {:?} edges", graph.len());

    // speeds things up a bit by tweaking edge directions.
    // loses the src < dst property, which can be helpful.
    redirect(graph);
    graph.sort();
    println!("graph data dirctd; {:?} edges", graph.len());
}

fn extract_fragment(graph: &Vec<(u32, u32)>, index: u64, parts: u64) -> GraphVector<u32> {
    let mut nodes = Vec::new();
    let mut edges = Vec::new();

    for &(src,dst) in graph {
        if src as u64 % parts == index {
            while src + 1 >= nodes.len() as u32 { nodes.push(0); }
            while dst + 1 >= nodes.len() as u32 { nodes.push(0); } // allows unsafe access to nodes

            nodes[src as usize + 1] += 1;
            edges.push(dst);
        }
    }

    for index in (1..nodes.len()) {
        nodes[index] += nodes[index - 1];
    }

    return GraphVector { nodes: nodes, edges: edges };
}

fn _digest_graph_vector<E: Ord+Copy>(graph: &GraphVector<E>, output_prefix: &str) {
    let mut edge_writer = BufWriter::new(File::create(format!("{}.targets", output_prefix)).unwrap());
    let mut node_writer = BufWriter::new(File::create(format!("{}.offsets", output_prefix)).unwrap());
    node_writer.write_all(unsafe { _typed_as_byte_slice(&graph.nodes[..]) }).unwrap();

    let mut slice = unsafe { _typed_as_byte_slice(&graph.edges[..]) };
    while slice.len() > 0 {
        let to_write = if slice.len() < 1000000 { slice.len() } else { 1000000 };
        edge_writer.write_all(&slice[..to_write]).unwrap();
        println!("wrote some; remaining: {}", slice.len());
        slice = &slice[to_write..];
    }
}

unsafe fn _typed_as_byte_slice<T>(slice: &[T]) -> &[u8] {
    std::mem::transmute(RawSlice {
        data: slice.as_ptr() as *const u8,
        len: slice.len() * std::mem::size_of::<T>(),
    })
}
<commit_msg>updated to new timely drop on github<commit_after>#![feature(core)]
#![feature(str_words)]

extern crate mmap;
extern crate time;
extern crate core;
extern crate timely;
extern crate columnar;
extern crate dataflow_join;

extern crate docopt;
use docopt::Docopt;

use std::io::{BufRead, BufReader, BufWriter, Write, stdin};
use std::fs::File;
use std::rc::Rc;
use std::cell::RefCell;
use std::thread;
use core::raw::Slice as RawSlice;

use dataflow_join::GenericJoinExt;
use dataflow_join::graph::{GraphTrait, GraphVector, GraphMMap, GraphExtenderExt, gallop};

use timely::progress::Graph;
use timely::progress::scope::Scope;
use timely::progress::nested::subgraph::new_graph;
use timely::progress::nested::product::Product;
use timely::example::*;
use timely::communication::*;

static USAGE: &'static str = "
Usage: triangles dataflow (text | binary) <source> <workers> [--inspect] [--interactive]
       triangles compute (text | binary) <source>
       triangles digest <source> <target>
       triangles help
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    if args.get_bool("dataflow") {
        let inspect = args.get_bool("--inspect");
        let interactive = args.get_bool("--interactive");
        let workers = if let Ok(threads) = args.get_str("<workers>").parse() { threads }
                      else { panic!("invalid setting for workers: {}", args.get_str("-t")) };;
        println!("starting triangles dataflow with {:?} worker{}; inspection: {:?}, interactive: {:?}", workers, if workers == 1 { "" } else { "s" }, inspect, interactive);
        let source = args.get_str("<source>");
        if args.get_bool("text") {
            let mut graph = livejournal(&source);
            organize_graph(&mut graph);
            triangles_multi(ProcessCommunicator::new_vector(workers), |index, peers| extract_fragment(&graph, index, peers), inspect, interactive);
        }
        if args.get_bool("binary") {
            triangles_multi(ProcessCommunicator::new_vector(workers), |_, _| GraphMMap::new(&source), inspect, interactive);
        }
    }
    if args.get_bool("compute") {
        let source = args.get_str("<source>");
        if args.get_bool("text") {
            let mut graph = livejournal(&source);
            organize_graph(&mut graph);
            let graph = extract_fragment(&graph, 0, 1);
            println!("triangles: {:?}", raw_triangles(&graph));
        }
        if args.get_bool("binary") {
            let graph = GraphMMap::new(&source);
            println!("triangles: {:?}", raw_triangles(&graph));
        }
    }
    if args.get_bool("digest") {
        println!("digest will overwrite <target>.targets and <target>.offsets, so careful");
        println!("at least, it will once you edit the code to uncomment the line.");
        let source = args.get_str("<source>");
        let _target = args.get_str("<target>");
        let mut graph = livejournal(&source);
        organize_graph(&mut graph);
        // _digest_graph_vector(&extract_fragment(&graph, 0, 1), _target); // will overwrite "prefix.offsets" and "prefix.targets"
    }
    if args.get_bool("help") {
        println!("the code presently assumes you have access to the livejournal graph, from:");
        println!("   https://snap.stanford.edu/data/soc-LiveJournal1.html");
        println!("");
        println!("stash that somewhere, and use it as text source for compute or digest.");
        println!("digest will overwrite <target>.targets and <target>.offsets, so careful");
        println!("at least, it will once you edit the code to uncomment the line.")
    }
}

fn intersect<E: Ord>(aaa: &[E], mut bbb: &[E]) -> u64 {
    let mut count = 0;
    for a in aaa {
        bbb = gallop(bbb, a);
        if bbb.len() > 0 && &bbb[0] == a { count += 1; }
    }
    count
}

fn raw_triangles<G: GraphTrait<Target=u32>>(graph: &G) -> u64 {
    let mut count = 0;
    for a in (0..graph.nodes()) {
        let aaa = graph.edges(a);
        for &b in aaa {
            let bbb = graph.edges(b as usize);
            count += if aaa.len() < bbb.len() { intersect(aaa.clone(), bbb) }
                     else                     { intersect(bbb, aaa.clone()) };
        }
    }
    count
}

fn triangles_multi<C: Communicator+Send, G: GraphTrait<Target=u32>, F: Fn(u64, u64)->G+Send+Sync>(communicators: Vec<C>, loader: F, inspect: bool, interactive: bool) {
    let mut guards = Vec::new();
    let loader = &loader;
    for communicator in communicators.into_iter() {
        guards.push(thread::Builder::new().name(format!("worker thread {}", communicator.index()))
                                          .scoped(move || triangles(communicator, loader, inspect, interactive))
                                          .unwrap());
    }
}

fn triangles<C: Communicator, G: GraphTrait<Target=u32>, F: Fn(u64, u64)->G>(communicator: C, loader: &F, inspect: bool, interactive: bool) {
    let comm_index = communicator.index();
    let comm_peers = communicator.peers();

    let graph = Rc::new(RefCell::new(loader(comm_index, comm_peers)));
    let mut computation = new_graph::<u64, C>(communicator);
    let (mut input, mut stream) = computation.new_input::<u32>();

    // // extend u32s to pairs, then pairs to triples.
    let mut triangles = stream.extend(vec![&graph.extend_using(|| { |&a| a as u64 } )])
                              .flat_map(|(p,es)| es.into_iter().map(move |e| (p.clone(), e)))
                              .extend(vec![&graph.extend_using(|| { |&(a,_)| a as u64 }),
                                           &graph.extend_using(|| { |&(_,b)| b as u64 })]);
                            //   .flat_map(|(p,es)| es.into_iter().map(move |e| (p.clone(), e)));

    // let mut triangles = _quads(&mut triangles, &graph);

     if inspect {
         triangles.inspect(|x| println!("triangles: {:?}", x));
     }

    computation.0.borrow_mut().get_internal_summary();
    computation.0.borrow_mut().set_external_summary(Vec::new(), &mut[]);

    if interactive {
        let mut stdinput = stdin();
        let mut line = String::new();
        for index in (0..) {
            stdinput.read_line(&mut line).unwrap();
            let start = time::precise_time_ns();
            if let Some(word) = line.words().next() {
                let read = word.parse();
                if let Ok(number) = read {
                    input.send_messages(&Product::new((), index as u64), vec![number]);
                }
            }
            line.clear();

            input.advance(&Product::new((), index as u64), &Product::new((), index as u64 + 1));
            for _ in (0..5) {
                computation.0.borrow_mut().pull_internal_progress(&mut[], &mut[], &mut[]);
            }

            println!("elapsed: {:?}us", (time::precise_time_ns() - start)/1000);
        }
    }
    else {
        let step = 1000;
        let nodes = graph.borrow().nodes() - 1;
        let limit = (nodes / step) + 1;
        for index in (0..limit) {
            let index = index as usize;
            let data = ((index * step) as u32 .. ((index + 1) * step) as u32).filter(|&x| x as u64 % comm_peers == comm_index)
                                                                             .filter(|&x| x < nodes as u32)
                                                                             .collect();
            input.send_messages(&Product::new((), index as u64), data);
            input.advance(&Product::new((), index as u64), &Product::new((), index as u64 + 1));
            computation.0.borrow_mut().pull_internal_progress(&mut[], &mut[], &mut[]);
        }
        input.close_at(&Product::new((), limit as u64));
        while computation.0.borrow_mut().pull_internal_progress(&mut[], &mut[], &mut[]) { }
    }
}

fn _quads<G: Graph, G2: GraphTrait<Target=u32>>(stream: &mut Stream<G, ((u32, u32), u32)>, graph: &Rc<RefCell<G2>>) -> Stream<G, (((u32, u32), u32), u32)> {
    //
    stream.extend(vec![&graph.extend_using(|| { |&((a,_),_)| a as u64 }),
                       &graph.extend_using(|| { |&((_,b),_)| b as u64 }),
                       &graph.extend_using(|| { |&((_,_),c)| c as u64 })])
          .flat_map(|(p,es)| es.into_iter().map(move |e| (p.clone(), e)))
    //
    //  let mut fives = quads.extend(vec![Box::new(fragment.extend_using(|| { |&(((a,_),_),_)| a as u64 })),
    //                                    Box::new(fragment.extend_using(|| { |&(((_,b),_),_)| b as u64 })),
    //                                    Box::new(fragment.extend_using(|| { |&(((_,_),c),_)| c as u64 })),
    //                                    Box::new(fragment.extend_using(|| { |&(((_,_),_),d)| d as u64 }))]).flatten();
    //
    // let mut sixes = fives.extend(vec![Box::new(fragment.extend_using(|| { |&((((a,_),_),_),_)| a as u64 })),
    //                                   Box::new(fragment.extend_using(|| { |&((((_,b),_),_),_)| b as u64 })),
    //                                   Box::new(fragment.extend_using(|| { |&((((_,_),c),_),_)| c as u64 })),
    //                                   Box::new(fragment.extend_using(|| { |&((((_,_),_),d),_)| d as u64 })),
    //                                   Box::new(fragment.extend_using(|| { |&((((_,_),_),_),e)| e as u64 }))]).flatten();
    //
    // sixes.observe();
}

fn redirect(edges: &mut Vec<(u32, u32)>) {
    let mut degrees = Vec::new();
    for &(src, dst) in edges.iter() {
        while src as usize >= degrees.len() { degrees.push(0); }
        while dst as usize >= degrees.len() { degrees.push(0); }
        degrees[src as usize] += 1;
        degrees[dst as usize] += 1;
    }

    for index in (0..edges.len()) {
        if degrees[edges[index].0 as usize] > degrees[edges[index].1 as usize] {
            edges[index] = (edges[index].1, edges[index].0);
        }
    }
}

// loads the livejournal file available at https://snap.stanford.edu/data/soc-LiveJournal1.html
fn livejournal(filename: &str) -> Vec<(u32, u32)> {
    let mut graph = Vec::new();
    let file = BufReader::new(File::open(filename).unwrap());
    for readline in file.lines() {
        let line = readline.ok().expect("read error");
        if !line.starts_with('#') {
            let elts: Vec<&str> = line[..].words().collect();
            let src: u32 = elts[0].parse().ok().expect("malformed src");
            let dst: u32 = elts[1].parse().ok().expect("malformed dst");
            if src < dst { graph.push((src, dst)) }
            if src > dst { graph.push((dst, src)) }
        }
    }

    println!("graph data loaded; {:?} edges", graph.len());
    return graph;
}

fn organize_graph(graph: &mut Vec<(u32, u32)>) {

    for index in (0..graph.len()) {
        if graph[index].1 < graph[index].0 {
            graph[index] = (graph[index].1, graph[index].0);
        }
    }

    graph.sort();
    println!("graph data sorted; {:?} edges", graph.len());

    graph.dedup();
    println!("graph data uniqed; {:?} edges", graph.len());

    // speeds things up a bit by tweaking edge directions.
    // loses the src < dst property, which can be helpful.
    redirect(graph);
    graph.sort();
    println!("graph data dirctd; {:?} edges", graph.len());
}

fn extract_fragment(graph: &Vec<(u32, u32)>, index: u64, parts: u64) -> GraphVector<u32> {
    let mut nodes = Vec::new();
    let mut edges = Vec::new();

    for &(src,dst) in graph {
        if src as u64 % parts == index {
            while src + 1 >= nodes.len() as u32 { nodes.push(0); }
            while dst + 1 >= nodes.len() as u32 { nodes.push(0); } // allows unsafe access to nodes

            nodes[src as usize + 1] += 1;
            edges.push(dst);
        }
    }

    for index in (1..nodes.len()) {
        nodes[index] += nodes[index - 1];
    }

    return GraphVector { nodes: nodes, edges: edges };
}

fn _digest_graph_vector<E: Ord+Copy>(graph: &GraphVector<E>, output_prefix: &str) {
    let mut edge_writer = BufWriter::new(File::create(format!("{}.targets", output_prefix)).unwrap());
    let mut node_writer = BufWriter::new(File::create(format!("{}.offsets", output_prefix)).unwrap());
    node_writer.write_all(unsafe { _typed_as_byte_slice(&graph.nodes[..]) }).unwrap();

    let mut slice = unsafe { _typed_as_byte_slice(&graph.edges[..]) };
    while slice.len() > 0 {
        let to_write = if slice.len() < 1000000 { slice.len() } else { 1000000 };
        edge_writer.write_all(&slice[..to_write]).unwrap();
        println!("wrote some; remaining: {}", slice.len());
        slice = &slice[to_write..];
    }
}

unsafe fn _typed_as_byte_slice<T>(slice: &[T]) -> &[u8] {
    std::mem::transmute(RawSlice {
        data: slice.as_ptr() as *const u8,
        len: slice.len() * std::mem::size_of::<T>(),
    })
}
<|endoftext|>