<commit_before>#![deny(intra_doc_link_resolution_failure)]

extern crate proc_macro;
extern crate protobuf;

use std::collections::hash_map::HashMap;
use std::fmt::Write as FmtWrite;
use std::fs::File;
use std::io;
use std::io::Write;
use std::path::{Path, PathBuf};

use protobuf::descriptor::*;
use protobuf::Message;

use protobuf::prelude::*;

mod amend_io_error_util;
pub mod case_convert;
mod compiler_plugin;
mod customize;
mod enums;
mod extensions;
mod field;
mod file;
pub(crate) mod file_and_mod;
mod file_descriptor;
mod inside;
mod map;
mod message;
mod oneof;
mod protobuf_abs_path;
mod protobuf_ident;
mod protobuf_rel_path;
mod rust_name;
mod rust_types_values;
mod serde;
mod well_known_types;

pub(crate) mod rust;
pub(crate) mod scope;
pub(crate) mod strx;
pub(crate) mod syntax;

use customize::customize_from_rustproto_for_file;
pub use customize::Customize;

pub mod code_writer;

use self::code_writer::CodeWriter;
use self::enums::*;
use self::extensions::*;
use self::message::*;
#[doc(hidden)]
pub use amend_io_error_util::amend_io_error;
use file::proto_path_to_rust_mod;
use map::map_entry;
use scope::FileScope;
use scope::RootScope;

use inside::protobuf_crate_path;
pub use protobuf_abs_path::ProtobufAbsolutePath;
pub use protobuf_ident::ProtobufIdent;
pub use protobuf_rel_path::ProtobufRelativePath;

fn escape_byte(s: &mut String, b: u8) {
    if b == b'\n' {
        write!(s, "\\n").unwrap();
    } else if b == b'\r' {
        write!(s, "\\r").unwrap();
    } else if b == b'\t' {
        write!(s, "\\t").unwrap();
    } else if b == b'\\' || b == b'"' {
        write!(s, "\\{}", b as char).unwrap();
    } else if b == b'\0' {
        write!(s, "\\0").unwrap();
    // ASCII printable except space
    } else if b > 0x20 && b < 0x7f {
        write!(s, "{}", b as char).unwrap();
    } else {
        write!(s, "\\x{:02x}", b).unwrap();
    }
}

fn write_file_descriptor_data(
    file: &FileDescriptorProto,
    customize: &Customize,
    w: &mut CodeWriter,
) {
    let fdp_bytes = file.write_to_bytes().unwrap();
    w.write_line("static file_descriptor_proto_data: &'static [u8] = b\"\\");
    w.indented(|w| {
        const MAX_LINE_LEN: usize = 72;

        let mut s = String::new();
        for &b in &fdp_bytes {
            let prev_len = s.len();
            escape_byte(&mut s, b);
            let truncate = s.len() > MAX_LINE_LEN;
            if truncate {
                s.truncate(prev_len);
            }
            if truncate || s.len() == MAX_LINE_LEN {
                write!(s, "\\").unwrap();
                w.write_line(&s);
                s.clear();
            }
            if truncate {
                escape_byte(&mut s, b);
            }
        }
        if !s.is_empty() {
            write!(s, "\\").unwrap();
            w.write_line(&s);
            s.clear();
        }
    });
    w.write_line("\";");
    w.write_line("");
    w.def_fn(
        &format!(
            "parse_descriptor_proto() -> {}::descriptor::FileDescriptorProto",
            protobuf_crate_path(customize)
        ),
        |w| {
            w.write_line(&format!(
                "{}::parse_from_bytes(file_descriptor_proto_data).unwrap()",
                protobuf_crate_path(customize)
            ));
        },
    );
    w.write_line("");
    w.write_line("/// `FileDescriptorProto` object which was a source for this generated file");
    w.pub_fn(
        &format!(
            "file_descriptor_proto() -> &'static {}::descriptor::FileDescriptorProto",
            protobuf_crate_path(customize)
        ),
        |w| {
            w.lazy_static(
                "file_descriptor_proto_lazy",
                &format!(
                    "{}::descriptor::FileDescriptorProto",
                    protobuf_crate_path(customize)
                ),
                protobuf_crate_path(customize),
            );
            w.block("file_descriptor_proto_lazy.get(|| {", "})", |w| {
                w.write_line("parse_descriptor_proto()");
            });
        },
    );
}

fn gen_file(
    file: &FileDescriptorProto,
    _files_map: &HashMap<&Path, &FileDescriptorProto>,
    root_scope: &RootScope,
    customize: &Customize,
    parser: &str,
) -> Option<compiler_plugin::GenResult> {
    // TODO: use it
    let mut customize = customize.clone();
    // options specified in invocation have precedence over options specified in file
    customize.update_with(&customize_from_rustproto_for_file(
        file.options.get_message(),
    ));

    let scope = FileScope {
        file_descriptor: file,
    }
    .to_scope();
    let lite_runtime = customize.lite_runtime.unwrap_or_else(|| {
        file.options.get_message().get_optimize_for() == file_options::OptimizeMode::LITE_RUNTIME
    });

    let mut v = Vec::new();

    {
        let mut w = CodeWriter::new(&mut v);

        w.write_generated_by("rust-protobuf", env!("CARGO_PKG_VERSION"), parser);

        w.write_line("");
        w.write_line(&format!("//! Generated file from `{}`", file.get_name()));
        if customize.inside_protobuf != Some(true) {
            w.write_line("");
            w.write_line("/// Generated files are compatible only with the same version");
            w.write_line("/// of protobuf runtime.");
            w.write_line(&format!(
                "const _PROTOBUF_VERSION_CHECK: () = {}::{};",
                protobuf_crate_path(&customize),
                protobuf::VERSION_IDENT
            ));
        }

        static NESTED_TYPE_NUMBER: protobuf::rt::LazyV2<i32> = protobuf::rt::LazyV2::INIT;
        let message_type_number = *NESTED_TYPE_NUMBER.get(|| {
            protobuf::reflect::MessageDescriptor::for_type::<FileDescriptorProto>()
                .get_field_by_name("message_type")
                .expect("`message_type` must exist")
                .proto()
                .get_number()
        });

        let mut path = vec![message_type_number, 0];
        for (id, message) in scope.get_messages().iter().enumerate() {
            // ignore map entries, because they are not used in map fields
            if map_entry(message).is_none() {
                path[1] = id as i32;

                w.write_line("");
                MessageGen::new(
                    message,
                    &root_scope,
                    &customize,
                    &path,
                    file.source_code_info.as_ref(),
                )
                .write(&mut w);
            }
        }

        static ENUM_TYPE_NUMBER: protobuf::rt::LazyV2<i32> = protobuf::rt::LazyV2::INIT;
        let enum_type_number = *ENUM_TYPE_NUMBER.get(|| {
            protobuf::reflect::MessageDescriptor::for_type::<FileDescriptorProto>()
                .get_field_by_name("enum_type")
                .expect("`enum_type` must exist")
                .proto()
                .get_number()
        });

        let mut path = vec![enum_type_number, 0];
        for (id, enum_type) in scope.get_enums().iter().enumerate() {
            path[1] = id as i32;

            w.write_line("");
            EnumGen::new(
                enum_type,
                &customize,
                root_scope,
                &path,
                file.source_code_info.as_ref(),
            )
            .write(&mut w);
        }

        write_extensions(file, &root_scope, &mut w, &customize);

        if !lite_runtime {
            w.write_line("");
            write_file_descriptor_data(file, &customize, &mut w);
        }
    }

    Some(compiler_plugin::GenResult {
        name: format!("{}.rs", proto_path_to_rust_mod(file.get_name())),
        content: v,
    })
}

// This function is also used externally by cargo plugin
// https://github.com/plietar/rust-protobuf-build
// So be careful changing its signature.
pub fn gen(
    file_descriptors: &[FileDescriptorProto],
    parser: &str,
    files_to_generate: &[PathBuf],
    customize: &Customize,
) -> Vec<compiler_plugin::GenResult> {
    let root_scope = RootScope {
        file_descriptors: file_descriptors,
    };

    let mut results: Vec<compiler_plugin::GenResult> = Vec::new();
    let files_map: HashMap<&Path, &FileDescriptorProto> = file_descriptors
        .iter()
        .map(|f| (Path::new(f.get_name()), f))
        .collect();

    for file_name in files_to_generate {
        let file = files_map.get(file_name.as_path()).expect(&format!(
            "file not found in file descriptors: {:?}, files: {:?}",
            file_name,
            files_map.keys()
        ));
        results.extend(gen_file(file, &files_map, &root_scope, customize, parser));
    }
    results
}

pub fn gen_and_write(
    file_descriptors: &[FileDescriptorProto],
    parser: &str,
    files_to_generate: &[PathBuf],
    out_dir: &Path,
    customize: &Customize,
) -> io::Result<()> {
    match out_dir.metadata() {
        Ok(m) => {
            if !m.is_dir() {
                return Err(io::Error::new(
                    io::ErrorKind::Other,
                    format!("{} is not a directory", out_dir.display()),
                ));
            }
        }
        Err(e) => {
            return Err(amend_io_error(
                e,
                format!("{} does not exist or not accessible", out_dir.display()),
            ));
        }
    }

    let results = gen(file_descriptors, parser, files_to_generate, customize);

    for r in &results {
        let mut file_path = out_dir.to_owned();
        file_path.push(&r.name);
        let mut file_writer = File::create(&file_path)
            .map_err(|e| amend_io_error(e, format!("failed to create {:?}", file_path)))?;
        file_writer
            .write_all(&r.content)
            .map_err(|e| amend_io_error(e, format!("failed to write to {:?}", file_path)))?;
        file_writer
            .flush()
            .map_err(|e| amend_io_error(e, format!("failed to flush {:?}", file_path)))?;
    }

    Ok(())
}

pub fn protoc_gen_rust_main() {
    compiler_plugin::plugin_main(|r| {
        let customize = Customize::parse_from_parameter(r.parameter).expect("parse options");
        gen(
            r.file_descriptors,
            "protoc --rust-out=...",
            r.files_to_generate,
            &customize,
        )
    });
}

/// Used in protobuf-codegen-identical-test
#[doc(hidden)]
pub fn proto_name_to_rs(name: &str) -> String {
    format!("{}.rs", proto_path_to_rust_mod(name))
}
<commit_msg>Inline parse_descriptor_proto function in generated code<commit_after>#![deny(intra_doc_link_resolution_failure)]

extern crate proc_macro;
extern crate protobuf;

use std::collections::hash_map::HashMap;
use std::fmt::Write as FmtWrite;
use std::fs::File;
use std::io;
use std::io::Write;
use std::path::{Path, PathBuf};

use protobuf::descriptor::*;
use protobuf::Message;

use protobuf::prelude::*;

mod amend_io_error_util;
pub mod case_convert;
mod compiler_plugin;
mod customize;
mod enums;
mod extensions;
mod field;
mod file;
pub(crate) mod file_and_mod;
mod file_descriptor;
mod inside;
mod map;
mod message;
mod oneof;
mod protobuf_abs_path;
mod protobuf_ident;
mod protobuf_rel_path;
mod rust_name;
mod rust_types_values;
mod serde;
mod well_known_types;

pub(crate) mod rust;
pub(crate) mod scope;
pub(crate) mod strx;
pub(crate) mod syntax;

use customize::customize_from_rustproto_for_file;
pub use customize::Customize;

pub mod code_writer;

use self::code_writer::CodeWriter;
use self::enums::*;
use self::extensions::*;
use self::message::*;
#[doc(hidden)]
pub use amend_io_error_util::amend_io_error;
use file::proto_path_to_rust_mod;
use map::map_entry;
use scope::FileScope;
use scope::RootScope;

use inside::protobuf_crate_path;
pub use protobuf_abs_path::ProtobufAbsolutePath;
pub use protobuf_ident::ProtobufIdent;
pub use protobuf_rel_path::ProtobufRelativePath;

fn escape_byte(s: &mut String, b: u8) {
    if b == b'\n' {
        write!(s, "\\n").unwrap();
    } else if b == b'\r' {
        write!(s, "\\r").unwrap();
    } else if b == b'\t' {
        write!(s, "\\t").unwrap();
    } else if b == b'\\' || b == b'"' {
        write!(s, "\\{}", b as char).unwrap();
    } else if b == b'\0' {
        write!(s, "\\0").unwrap();
    // ASCII printable except space
    } else if b > 0x20 && b < 0x7f {
        write!(s, "{}", b as char).unwrap();
    } else {
        write!(s, "\\x{:02x}", b).unwrap();
    }
}

fn write_file_descriptor_data(
    file: &FileDescriptorProto,
    customize: &Customize,
    w: &mut CodeWriter,
) {
    let fdp_bytes = file.write_to_bytes().unwrap();
    w.write_line("static file_descriptor_proto_data: &'static [u8] = b\"\\");
    w.indented(|w| {
        const MAX_LINE_LEN: usize = 72;

        let mut s = String::new();
        for &b in &fdp_bytes {
            let prev_len = s.len();
            escape_byte(&mut s, b);
            let truncate = s.len() > MAX_LINE_LEN;
            if truncate {
                s.truncate(prev_len);
            }
            if truncate || s.len() == MAX_LINE_LEN {
                write!(s, "\\").unwrap();
                w.write_line(&s);
                s.clear();
            }
            if truncate {
                escape_byte(&mut s, b);
            }
        }
        if !s.is_empty() {
            write!(s, "\\").unwrap();
            w.write_line(&s);
            s.clear();
        }
    });
    w.write_line("\";");
    w.write_line("");
    w.write_line("/// `FileDescriptorProto` object which was a source for this generated file");
    w.pub_fn(
        &format!(
            "file_descriptor_proto() -> &'static {}::descriptor::FileDescriptorProto",
            protobuf_crate_path(customize)
        ),
        |w| {
            w.lazy_static(
                "file_descriptor_proto_lazy",
                &format!(
                    "{}::descriptor::FileDescriptorProto",
                    protobuf_crate_path(customize)
                ),
                protobuf_crate_path(customize),
            );
            w.block("file_descriptor_proto_lazy.get(|| {", "})", |w| {
                w.write_line(&format!(
                    "{}::parse_from_bytes(file_descriptor_proto_data).unwrap()",
                    protobuf_crate_path(customize)
                ));
            });
        },
    );
}

fn gen_file(
    file: &FileDescriptorProto,
    _files_map: &HashMap<&Path, &FileDescriptorProto>,
    root_scope: &RootScope,
    customize: &Customize,
    parser: &str,
) -> Option<compiler_plugin::GenResult> {
    // TODO: use it
    let mut customize = customize.clone();
    // options specified in invocation have precedence over options specified in file
    customize.update_with(&customize_from_rustproto_for_file(
        file.options.get_message(),
    ));

    let scope = FileScope {
        file_descriptor: file,
    }
    .to_scope();
    let lite_runtime = customize.lite_runtime.unwrap_or_else(|| {
        file.options.get_message().get_optimize_for() == file_options::OptimizeMode::LITE_RUNTIME
    });

    let mut v = Vec::new();

    {
        let mut w = CodeWriter::new(&mut v);

        w.write_generated_by("rust-protobuf", env!("CARGO_PKG_VERSION"), parser);

        w.write_line("");
        w.write_line(&format!("//! Generated file from `{}`", file.get_name()));
        if customize.inside_protobuf != Some(true) {
            w.write_line("");
            w.write_line("/// Generated files are compatible only with the same version");
            w.write_line("/// of protobuf runtime.");
            w.write_line(&format!(
                "const _PROTOBUF_VERSION_CHECK: () = {}::{};",
                protobuf_crate_path(&customize),
                protobuf::VERSION_IDENT
            ));
        }

        static NESTED_TYPE_NUMBER: protobuf::rt::LazyV2<i32> = protobuf::rt::LazyV2::INIT;
        let message_type_number = *NESTED_TYPE_NUMBER.get(|| {
            protobuf::reflect::MessageDescriptor::for_type::<FileDescriptorProto>()
                .get_field_by_name("message_type")
                .expect("`message_type` must exist")
                .proto()
                .get_number()
        });

        let mut path = vec![message_type_number, 0];
        for (id, message) in scope.get_messages().iter().enumerate() {
            // ignore map entries, because they are not used in map fields
            if map_entry(message).is_none() {
                path[1] = id as i32;

                w.write_line("");
                MessageGen::new(
                    message,
                    &root_scope,
                    &customize,
                    &path,
                    file.source_code_info.as_ref(),
                )
                .write(&mut w);
            }
        }

        static ENUM_TYPE_NUMBER: protobuf::rt::LazyV2<i32> = protobuf::rt::LazyV2::INIT;
        let enum_type_number = *ENUM_TYPE_NUMBER.get(|| {
            protobuf::reflect::MessageDescriptor::for_type::<FileDescriptorProto>()
                .get_field_by_name("enum_type")
                .expect("`enum_type` must exist")
                .proto()
                .get_number()
        });

        let mut path = vec![enum_type_number, 0];
        for (id, enum_type) in scope.get_enums().iter().enumerate() {
            path[1] = id as i32;

            w.write_line("");
            EnumGen::new(
                enum_type,
                &customize,
                root_scope,
                &path,
                file.source_code_info.as_ref(),
            )
            .write(&mut w);
        }

        write_extensions(file, &root_scope, &mut w, &customize);

        if !lite_runtime {
            w.write_line("");
            write_file_descriptor_data(file, &customize, &mut w);
        }
    }

    Some(compiler_plugin::GenResult {
        name: format!("{}.rs", proto_path_to_rust_mod(file.get_name())),
        content: v,
    })
}

// This function is also used externally by cargo plugin
// https://github.com/plietar/rust-protobuf-build
// So be careful changing its signature.
pub fn gen(
    file_descriptors: &[FileDescriptorProto],
    parser: &str,
    files_to_generate: &[PathBuf],
    customize: &Customize,
) -> Vec<compiler_plugin::GenResult> {
    let root_scope = RootScope {
        file_descriptors: file_descriptors,
    };

    let mut results: Vec<compiler_plugin::GenResult> = Vec::new();
    let files_map: HashMap<&Path, &FileDescriptorProto> = file_descriptors
        .iter()
        .map(|f| (Path::new(f.get_name()), f))
        .collect();

    for file_name in files_to_generate {
        let file = files_map.get(file_name.as_path()).expect(&format!(
            "file not found in file descriptors: {:?}, files: {:?}",
            file_name,
            files_map.keys()
        ));
        results.extend(gen_file(file, &files_map, &root_scope, customize, parser));
    }
    results
}

pub fn gen_and_write(
    file_descriptors: &[FileDescriptorProto],
    parser: &str,
    files_to_generate: &[PathBuf],
    out_dir: &Path,
    customize: &Customize,
) -> io::Result<()> {
    match out_dir.metadata() {
        Ok(m) => {
            if !m.is_dir() {
                return Err(io::Error::new(
                    io::ErrorKind::Other,
                    format!("{} is not a directory", out_dir.display()),
                ));
            }
        }
        Err(e) => {
            return Err(amend_io_error(
                e,
                format!("{} does not exist or not accessible", out_dir.display()),
            ));
        }
    }

    let results = gen(file_descriptors, parser, files_to_generate, customize);

    for r in &results {
        let mut file_path = out_dir.to_owned();
        file_path.push(&r.name);
        let mut file_writer = File::create(&file_path)
            .map_err(|e| amend_io_error(e, format!("failed to create {:?}", file_path)))?;
        file_writer
            .write_all(&r.content)
            .map_err(|e| amend_io_error(e, format!("failed to write to {:?}", file_path)))?;
        file_writer
            .flush()
            .map_err(|e| amend_io_error(e, format!("failed to flush {:?}", file_path)))?;
    }

    Ok(())
}

pub fn protoc_gen_rust_main() {
    compiler_plugin::plugin_main(|r| {
        let customize = Customize::parse_from_parameter(r.parameter).expect("parse options");
        gen(
            r.file_descriptors,
            "protoc --rust-out=...",
            r.files_to_generate,
            &customize,
        )
    });
}

/// Used in protobuf-codegen-identical-test
#[doc(hidden)]
pub fn proto_name_to_rs(name: &str) -> String {
    format!("{}.rs", proto_path_to_rust_mod(name))
}
<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

extern crate time;

use std::env;
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use std::process::Command;

fn main() {
    let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());

    File::create(out_dir.join("build-info.txt"))
        .unwrap()
        .write_all(build_info().as_bytes())
        .unwrap();
}

// build_info returns a string of commit hash and utc time.
// TODO: use serde.
fn build_info() -> String {
    // explicit separates outputs by '\n'.
    format!(
        "{}\n{}\n{}\n{}",
        commit_hash().trim_right(),
        branch_info().trim_right(),
        utc_time(),
        rustc_version()
    )
}

fn utc_time() -> String {
    let utc = time::now_utc();
    format!("{}", utc.strftime("%Y-%m-%d %I:%M:%S").unwrap())
}

fn commit_hash() -> String {
    let mut cmd = Command::new("git");
    cmd.args(&["rev-parse", "HEAD"]);
    cmd.output()
        .ok()
        .and_then(|o| String::from_utf8(o.stdout).ok())
        .unwrap_or("None".to_owned())
}

fn branch_info() -> String {
    let mut cmd = Command::new("git");
    cmd.args(&["rev-parse", "--abbrev-ref", "HEAD"]);
    cmd.output()
        .ok()
        .and_then(|o| String::from_utf8(o.stdout).ok())
        .unwrap_or("None".to_owned())
}

fn rustc_version() -> String {
    let mut cmd = Command::new(env::var("RUSTC").unwrap());
    cmd.args(&["--version"]);
    cmd.output()
        .ok()
        .and_then(|o| String::from_utf8(o.stdout).ok())
        .map(|v| v.trim_left_matches("rustc").trim().to_owned())
        .unwrap()
}
<commit_msg>fix build.rs: replace "%I" with "%H". (#2836)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

extern crate time;

use std::env;
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use std::process::Command;

fn main() {
    let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());

    File::create(out_dir.join("build-info.txt"))
        .unwrap()
        .write_all(build_info().as_bytes())
        .unwrap();
}

// build_info returns a string of commit hash and utc time.
// TODO: use serde.
fn build_info() -> String {
    // explicit separates outputs by '\n'.
    format!(
        "{}\n{}\n{}\n{}",
        commit_hash().trim_right(),
        branch_info().trim_right(),
        utc_time(),
        rustc_version()
    )
}

fn utc_time() -> String {
    let utc = time::now_utc();
    format!("{}", utc.strftime("%Y-%m-%d %H:%M:%S").unwrap())
}

fn commit_hash() -> String {
    let mut cmd = Command::new("git");
    cmd.args(&["rev-parse", "HEAD"]);
    cmd.output()
        .ok()
        .and_then(|o| String::from_utf8(o.stdout).ok())
        .unwrap_or("None".to_owned())
}

fn branch_info() -> String {
    let mut cmd = Command::new("git");
    cmd.args(&["rev-parse", "--abbrev-ref", "HEAD"]);
    cmd.output()
        .ok()
        .and_then(|o| String::from_utf8(o.stdout).ok())
        .unwrap_or("None".to_owned())
}

fn rustc_version() -> String {
    let mut cmd = Command::new(env::var("RUSTC").unwrap());
    cmd.args(&["--version"]);
    cmd.output()
        .ok()
        .and_then(|o| String::from_utf8(o.stdout).ok())
        .map(|v| v.trim_left_matches("rustc").trim().to_owned())
        .unwrap()
}
<|endoftext|>


<commit_before>// Compiles the `rsmpi` C shim library.
extern crate gcc;
// Generates the Rust header for the C API.
extern crate bindgen;
// Finds out information about the MPI library
extern crate build_probe_mpi;
// Inspect version of rustc compiler
extern crate rustc_version;

use std::env;
use std::path::Path;

fn main() {
    // Use `mpicc` wrapper rather than the system C compiler.
    env::set_var("CC", "mpicc");
    // Build the `rsmpi` C shim library.
    gcc::compile_library("librsmpi.a", &["src/ffi/rsmpi.c"]);

    // Try to find an MPI library
    let lib = match build_probe_mpi::probe() {
        Ok(lib) => lib,
        Err(errs) => {
            println!("Could not find MPI library for various reasons:\n");
            for (i, err) in errs.iter().enumerate() {
                println!("Reason #{}:\n{}\n", i, err);
            }
            panic!();
        }
    };

    // Let `rustc` know about the library search directories.
    for dir in &lib.lib_paths {
        println!("cargo:rustc-link-search=native={}", dir.display());
    }
    for lib in &lib.libs {
        println!("cargo:rustc-link-lib={}", lib);
    }

    let mut builder = bindgen::builder();
    // Let `bindgen` know about libraries and search directories.
    for lib in &lib.libs { builder = builder.link(lib.clone()); }
    for dir in &lib.lib_paths { builder = builder.clang_arg(format!("-L{}", dir.display())); }
    for dir in &lib.include_paths { builder = builder.clang_arg(format!("-I{}", dir.display())); }

    // Generate Rust bindings for the MPI C API.
    let bindings = builder
        .header("src/ffi/rsmpi.h")
        .emit_builtins()
        .no_unstable_rust()
        .generate()
        .unwrap();

    // Write the bindings to disk.
    let out_dir = env::var("OUT_DIR").expect("cargo did not set OUT_DIR");
    let out_file = Path::new(&out_dir).join("functions_and_types.rs");
    bindings
        .write_to_file(out_file)
        .unwrap();

    // Access to extern statics has to be marked unsafe after 1.13.0
    if rustc_version::version_matches(">=1.13.0") {
        println!("cargo:rustc-cfg=extern_statics_are_unsafe");
    }
}
<commit_msg>blacklist type containing long double<commit_after>// Compiles the `rsmpi` C shim library.
extern crate gcc;
// Generates the Rust header for the C API.
extern crate bindgen;
// Finds out information about the MPI library
extern crate build_probe_mpi;
// Inspect version of rustc compiler
extern crate rustc_version;

use std::env;
use std::path::Path;

fn main() {
    // Use `mpicc` wrapper rather than the system C compiler.
    env::set_var("CC", "mpicc");
    // Build the `rsmpi` C shim library.
    gcc::compile_library("librsmpi.a", &["src/ffi/rsmpi.c"]);

    // Try to find an MPI library
    let lib = match build_probe_mpi::probe() {
        Ok(lib) => lib,
        Err(errs) => {
            println!("Could not find MPI library for various reasons:\n");
            for (i, err) in errs.iter().enumerate() {
                println!("Reason #{}:\n{}\n", i, err);
            }
            panic!();
        }
    };

    // Let `rustc` know about the library search directories.
    for dir in &lib.lib_paths {
        println!("cargo:rustc-link-search=native={}", dir.display());
    }
    for lib in &lib.libs {
        println!("cargo:rustc-link-lib={}", lib);
    }

    let mut builder = bindgen::builder();
    // Let `bindgen` know about libraries and search directories.
    for lib in &lib.libs { builder = builder.link(lib.clone()); }
    for dir in &lib.lib_paths { builder = builder.clang_arg(format!("-L{}", dir.display())); }
    for dir in &lib.include_paths { builder = builder.clang_arg(format!("-I{}", dir.display())); }

    // Generate Rust bindings for the MPI C API.
    let bindings = builder
        .header("src/ffi/rsmpi.h")
        .emit_builtins()
        .no_unstable_rust()
        .hide_type("mpich_struct_mpi_long_double_int")
        .generate()
        .unwrap();

    // Write the bindings to disk.
    let out_dir = env::var("OUT_DIR").expect("cargo did not set OUT_DIR");
    let out_file = Path::new(&out_dir).join("functions_and_types.rs");
    bindings
        .write_to_file(out_file)
        .unwrap();

    // Access to extern statics has to be marked unsafe after 1.13.0
    if rustc_version::version_matches(">=1.13.0") {
        println!("cargo:rustc-cfg=extern_statics_are_unsafe");
    }
}
<|endoftext|>


<commit_before>extern crate gcc;

fn main() {
    gcc::compile_library("libsid.a", &["./src/obtain_sid.c"]);
}
<commit_msg>core crate compiles under linux now<commit_after>extern crate gcc;

#[cfg(windows)]
fn main() {
    gcc::compile_library("libsid.a", &["./src/obtain_sid.c"]);
}

#[cfg(not(windows))]
fn main() {}
<|endoftext|>


<commit_before>extern crate rustc_version;
extern crate rusoto_codegen;

use std::env;
use std::path::Path;
use std::io::Write;
use std::fs::File;
use std::thread;

use rusoto_codegen::{Service, generate};

/// Parses and generates variables used to construct a User-Agent.
///
/// This is used to create a User-Agent header string resembling
/// `rusoto/x.y.z rust/x.y.z <os>`.
fn generate_user_agent_vars(output_path: &Path) {
    let rust_version = rustc_version::version();
    let mut f = File::create(&output_path.join("user_agent_vars.rs"))
            .expect("Could not create user agent file");
    f.write_all(format!("static RUST_VERSION: &'static str = \"{}\";", rust_version).as_bytes())
            .expect("Unable to write user agent");
}

/*
gamelift/2015-10-01/service-2.json:    "protocol":"json"
support/2013-04-15/service-2.json:    "protocol":"json"
*/

// expand to use cfg!() so codegen only gets run for services
// in the features list
macro_rules! services {
    ( $( [$name:expr, $date:expr] ),* ) => {
        {
            let mut services = Vec::new();
            $(
                if cfg!(feature = $name) {
                    services.push(Service::new($name, $date));
                }
            )*
            services
        }
    }
}

fn main() {
    let out_dir = env::var_os("OUT_DIR").expect("OUT_DIR not specified");
    let out_path = Path::new(&out_dir).to_owned();

    let services = services! {
        ["acm", "2015-12-08"],
        ["cloudformation", "2010-05-15"],
        ["cloudfront", "2016-11-25"],
        ["cloudhsm", "2014-05-30"],
        ["cloudtrail", "2013-11-01"],
        ["cloudwatch", "2010-08-01"],
        ["codecommit", "2015-04-13"],
        ["codedeploy", "2014-10-06"],
        ["codepipeline", "2015-07-09"],
        ["cognito-identity", "2014-06-30"],
        ["config", "2014-11-12"],
        ["datapipeline", "2012-10-29"],
        ["devicefarm", "2015-06-23"],
        ["directconnect", "2012-10-25"],
        ["ds", "2015-04-16"],
        ["dynamodb", "2012-08-10"],
        ["dynamodbstreams", "2012-08-10"],
        ["ec2", "2015-10-01"],
        ["ecr", "2015-09-21"],
        ["ecs", "2014-11-13"],
        ["elastictranscoder", "2012-09-25"],
        ["emr", "2009-03-31"],
        ["events", "2014-02-03"],
        ["firehose", "2015-08-04"],
        ["iam", "2010-05-08"],
        ["inspector", "2016-02-16"],
        ["iot", "2015-05-28"],
        ["kinesis", "2013-12-02"],
        ["kms", "2014-11-01"],
        ["logs", "2014-03-28"],
        ["machinelearning", "2014-12-12"],
        ["marketplacecommerceanalytics", "2015-07-01"],
        ["opsworks", "2013-02-18"],
        ["route53", "2013-04-01"],
        ["route53domains", "2014-05-15"],
        ["s3", "2006-03-01"],
        ["sqs", "2012-11-05"],
        ["ssm", "2014-11-06"],
        ["storagegateway", "2013-06-30"],
        ["swf", "2012-01-25"],
        ["waf", "2015-08-24"],
        ["workspaces", "2015-04-08"]
    };

    let mut generate_threads = vec![];
    for service in services {
        let thread_outpath = out_path.clone();
        generate_threads.push(thread::spawn(move || {
            generate(service, &thread_outpath);
        }));
    }

    for child_thread in generate_threads {
        match child_thread.join() {
            Ok(_) => {},
            Err(err) => panic!(err)
        }
    }

    generate_user_agent_vars(&out_path);

    let codegen_dir = Path::new("codegen");

    // avoid unnecessary recompiles when used as a crates.io dependency
    if codegen_dir.exists() {
        println!("cargo:rerun-if-changed=codegen");
    }
}
<commit_msg>Updates events to latest service definition.<commit_after>extern crate rustc_version;
extern crate rusoto_codegen;

use std::env;
use std::path::Path;
use std::io::Write;
use std::fs::File;
use std::thread;

use rusoto_codegen::{Service, generate};

/// Parses and generates variables used to construct a User-Agent.
///
/// This is used to create a User-Agent header string resembling
/// `rusoto/x.y.z rust/x.y.z <os>`.
fn generate_user_agent_vars(output_path: &Path) {
    let rust_version = rustc_version::version();
    let mut f = File::create(&output_path.join("user_agent_vars.rs"))
            .expect("Could not create user agent file");
    f.write_all(format!("static RUST_VERSION: &'static str = \"{}\";", rust_version).as_bytes())
            .expect("Unable to write user agent");
}

/*
gamelift/2015-10-01/service-2.json:    "protocol":"json"
support/2013-04-15/service-2.json:    "protocol":"json"
*/

// expand to use cfg!() so codegen only gets run for services
// in the features list
macro_rules! services {
    ( $( [$name:expr, $date:expr] ),* ) => {
        {
            let mut services = Vec::new();
            $(
                if cfg!(feature = $name) {
                    services.push(Service::new($name, $date));
                }
            )*
            services
        }
    }
}

fn main() {
    let out_dir = env::var_os("OUT_DIR").expect("OUT_DIR not specified");
    let out_path = Path::new(&out_dir).to_owned();

    let services = services! {
        ["acm", "2015-12-08"],
        ["cloudformation", "2010-05-15"],
        ["cloudfront", "2016-11-25"],
        ["cloudhsm", "2014-05-30"],
        ["cloudtrail", "2013-11-01"],
        ["cloudwatch", "2010-08-01"],
        ["codecommit", "2015-04-13"],
        ["codedeploy", "2014-10-06"],
        ["codepipeline", "2015-07-09"],
        ["cognito-identity", "2014-06-30"],
        ["config", "2014-11-12"],
        ["datapipeline", "2012-10-29"],
        ["devicefarm", "2015-06-23"],
        ["directconnect", "2012-10-25"],
        ["ds", "2015-04-16"],
        ["dynamodb", "2012-08-10"],
        ["dynamodbstreams", "2012-08-10"],
        ["ec2", "2015-10-01"],
        ["ecr", "2015-09-21"],
        ["ecs", "2014-11-13"],
        ["elastictranscoder", "2012-09-25"],
        ["emr", "2009-03-31"],
        ["events", "2015-10-07"],
        ["firehose", "2015-08-04"],
        ["iam", "2010-05-08"],
        ["inspector", "2016-02-16"],
        ["iot", "2015-05-28"],
        ["kinesis", "2013-12-02"],
        ["kms", "2014-11-01"],
        ["logs", "2014-03-28"],
        ["machinelearning", "2014-12-12"],
        ["marketplacecommerceanalytics", "2015-07-01"],
        ["opsworks", "2013-02-18"],
        ["route53", "2013-04-01"],
        ["route53domains", "2014-05-15"],
        ["s3", "2006-03-01"],
        ["sqs", "2012-11-05"],
        ["ssm", "2014-11-06"],
        ["storagegateway", "2013-06-30"],
        ["swf", "2012-01-25"],
        ["waf", "2015-08-24"],
        ["workspaces", "2015-04-08"]
    };

    let mut generate_threads = vec![];
    for service in services {
        let thread_outpath = out_path.clone();
        generate_threads.push(thread::spawn(move || {
            generate(service, &thread_outpath);
        }));
    }

    for child_thread in generate_threads {
        match child_thread.join() {
            Ok(_) => {},
            Err(err) => panic!(err)
        }
    }

    generate_user_agent_vars(&out_path);

    let codegen_dir = Path::new("codegen");

    // avoid unnecessary recompiles when used as a crates.io dependency
    if codegen_dir.exists() {
        println!("cargo:rerun-if-changed=codegen");
    }
}
<|endoftext|>


<commit_before>extern crate pkg_config;
use std::process::Command;
use std::path::PathBuf;
use std::env;
use std::fs;

macro_rules! sio {
    ($expr:expr) => { format!("libsoundio-{}", $expr) }
}

macro_rules! err_exists {
    ($expr:expr, $msg:expr) => {
        match $expr {
            Ok(val) => val,
            Err(err) => {
                match err.kind() {
                    ::std::io::ErrorKind::AlreadyExists => (),
                    _ => panic!(format!("{}: {}", $msg, err)),
                }
            }
        }
    }
}

fn lib_available(name: &str) -> bool {
    match pkg_config::find_library(name) {
        Ok(_) => true,
        Err(_) => {
            let res = Command::new("ldconfig").arg("--print-cache").output().unwrap();
            if res.status.success() {
                String::from_utf8(res.stdout).unwrap().contains(name)
            } else {
                false
            }
        }
    }
}

fn windows(_: String) {
    // TODO: Use precompiled binary!
    unimplemented!();
}

fn osx(_: String) {
    // TODO: I down't own any apple hardware to test this.
    unimplemented!();
}

fn linux(target: String) {
    if !lib_available("soundio") {
        build(target);
    } else {
        // TODO: remove else branch
        build(target);
    }
}

fn main() {
    let target = env::var("TARGET").unwrap();
    if target.contains("windows") {
        windows(target)
    } else if target.contains("apple") {
        osx(target)
    } else {
        // assume the rest is linux
        linux(target)
    }
}

fn sio_url(ext: &'static str) -> String {
    format!("http://libsound.io/release/{}.{}", sio!("1.1.0"), ext)
}

fn build(target: String) {
    let host = env::var("HOST").unwrap();
    let dst_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
    let lib_dir = dst_dir.join("lib");
    let include_dir = dst_dir.join("include");
    err_exists!(fs::create_dir(&lib_dir), &lib_dir.display());
    err_exists!(fs::create_dir(&include_dir), &include_dir.display());

    // set cargo flags
    println!("cargo:rustc-link-search=native={}", &lib_dir.display()); // -L
    println!("cargo:include={}", &include_dir.display());
    println!("cargo:root={}", &dst_dir.display());

    // download and extract libsoundio source
    Command::new("curl")
        .current_dir(&dst_dir)
        .args(&["--location", "--remote-name"])
        .arg(sio_url("tar.gz"))
        .output()
        .unwrap();
    Command::new("tar")
        .current_dir(&dst_dir)
        .arg("-xvzf")
        .arg(format!("{}.{}", sio!("1.1.0"), "tar.gz"))
        .output()
        .unwrap();

    // create build dir
    let soundio_root = dst_dir.join(sio!("1.1.0"));
    let build_dir = soundio_root.join("build");
    err_exists!(fs::create_dir(&build_dir), &build_dir.display());

    // run cmake
    Command::new("cmake")
        .current_dir(&build_dir)
        .arg("-DCMAKE_BUILD_TYPE=Debug")
        .arg("-DCMAKE_INSTALL_LIBDIR:PATH=lib")
        .arg(format!("-DCMAKE_INSTALL_PREFIX:PATH={}", &dst_dir.display()))
        .arg("-DBUILD_EXAMPLE_PROGRAMS:BOOL=OFF")
        .arg("-DBUILD_TESTS:BOOL=OFF")
        .arg("-DBUILD_STATIC_LIBS:BOOL=OFF")
        .arg("-DBUILD_SHARED_LIBS:BOOL=ON")
        .arg("-DENABLE_JACK:BOOL=OFF")
        .arg("-DENABLE_PULSEAUDIO:BOOL=OFF")
        .arg("-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON")
        .arg("..")
        .output()
        .unwrap();

    // make install
    Command::new("make")
        .current_dir(&build_dir)
        .arg("install")
        .output()
        .unwrap();

    // remove builddir
    fs::remove_dir_all(build_dir).unwrap();
}
<commit_msg>Build as dynamic library.<commit_after>extern crate pkg_config;
use std::process::Command;
use std::path::PathBuf;
use std::env;
use std::fs;

macro_rules! sio {
    ($expr:expr) => { format!("libsoundio-{}", $expr) }
}

macro_rules! err_exists {
    ($expr:expr, $msg:expr) => {
        match $expr {
            Ok(val) => val,
            Err(err) => {
                match err.kind() {
                    ::std::io::ErrorKind::AlreadyExists => (),
                    _ => panic!(format!("{}: {}", $msg, err)),
                }
            }
        }
    }
}

fn lib_available(name: &str) -> bool {
    match pkg_config::find_library(name) {
        Ok(_) => true,
        Err(_) => {
            let res = Command::new("ldconfig").arg("--print-cache").output().unwrap();
            if res.status.success() {
                String::from_utf8(res.stdout).unwrap().contains(name)
            } else {
                false
            }
        }
    }
}

fn windows(_: String) {
    // TODO: Use precompiled binary!
    unimplemented!();
}

fn osx(_: String) {
    // TODO: I down't own any apple hardware to test this.
    unimplemented!();
}

fn linux(target: String) {
    if !lib_available("soundio") {
        build(target);
    } else {
        // TODO: remove else branch
        build(target);
    }
}

fn main() {
    let target = env::var("TARGET").unwrap();
    if target.contains("windows") {
        windows(target)
    } else if target.contains("apple") {
        osx(target)
    } else {
        // assume the rest is linux
        linux(target)
    }
}

fn sio_url(ext: &'static str) -> String {
    format!("http://libsound.io/release/{}.{}", sio!("1.1.0"), ext)
}

fn build(target: String) {
    let host = env::var("HOST").unwrap();
    let dst_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
    let lib_dir = dst_dir.join("lib");
    let include_dir = dst_dir.join("include");
    err_exists!(fs::create_dir(&lib_dir), &lib_dir.display());
    err_exists!(fs::create_dir(&include_dir), &include_dir.display());

    // set cargo flags
    println!("cargo:rustc-link-lib=dylib={}", "soundio"); // -l
    println!("cargo:rustc-link-search=native={}", &lib_dir.display()); // -L
    println!("cargo:include={}", &include_dir.display());
    println!("cargo:root={}", &dst_dir.display());

    // download and extract libsoundio source
    Command::new("curl")
        .current_dir(&dst_dir)
        .args(&["--location", "--remote-name"])
        .arg(sio_url("tar.gz"))
        .output()
        .unwrap();
    Command::new("tar")
        .current_dir(&dst_dir)
        .arg("-xvzf")
        .arg(format!("{}.{}", sio!("1.1.0"), "tar.gz"))
        .output()
        .unwrap();

    // create build dir
    let soundio_root = dst_dir.join(sio!("1.1.0"));
    let build_dir = soundio_root.join("build");
    err_exists!(fs::create_dir(&build_dir), &build_dir.display());

    // run cmake
    Command::new("cmake")
        .current_dir(&build_dir)
        .arg("-DCMAKE_BUILD_TYPE=Debug")
        .arg("-DCMAKE_INSTALL_LIBDIR:PATH=lib")
        .arg(format!("-DCMAKE_INSTALL_PREFIX:PATH={}", &dst_dir.display()))
        .arg("-DBUILD_EXAMPLE_PROGRAMS:BOOL=OFF")
        .arg("-DBUILD_TESTS:BOOL=OFF")
        .arg("-DBUILD_STATIC_LIBS:BOOL=OFF")
        .arg("-DBUILD_SHARED_LIBS:BOOL=ON")
        .arg("-DENABLE_JACK:BOOL=OFF")
        .arg("-DENABLE_PULSEAUDIO:BOOL=OFF")
        .arg("-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON")
        .arg("..")
        .output()
        .unwrap();

    // make install
    Command::new("make")
        .current_dir(&build_dir)
        .arg("install")
        .output()
        .unwrap();

    // remove builddir
    fs::remove_dir_all(build_dir).unwrap();
}
<|endoftext|>


<commit_before>extern crate amq_protocol_codegen;
extern crate amq_protocol_types;
extern crate handlebars;
extern crate serde_json;

use amq_protocol_codegen::*;
use amq_protocol_types::*;
use handlebars::{Handlebars,Helper,HelperDef,RenderContext,RenderError};

use std::collections::BTreeMap;
use std::env;
use std::fs::File;
use std::io::{Read, Write};
use std::path::Path;
use std::ascii::AsciiExt;

fn main() {
    let out_dir      = env::var("OUT_DIR").expect("OUT_DIR is not defined");
    let dest_path    = Path::new(&out_dir).join("generated.rs");
    let dest_path2   = Path::new(&out_dir).join("api.rs");
    let mut f        = File::create(&dest_path).expect("Failed to create generated.rs");
    let mut f2       = File::create(&dest_path2).expect("Failed to create api.rs");
    let mut full_tpl = String::new();
    let mut api_tpl = String::new();
    std::fs::File::open("templates/main.rs").expect("Failed to open main template").read_to_string(&mut full_tpl).expect("Failed to read main template");
    std::fs::File::open("templates/api.rs").expect("Failed to open main template").read_to_string(&mut api_tpl).expect("Failed to read main template");

    let mut handlebars = Handlebars::new();
    let mut data = BTreeMap::new();

    handlebars.register_escape_fn(handlebars::no_escape);
    handlebars.register_helper("snake", Box::new(snake_helper));
    handlebars.register_helper("camel", Box::new(camel_helper));
    handlebars.register_helper("map_type", Box::new(map_type_helper));
    handlebars.register_helper("map_parser", Box::new(map_parser_helper));
    handlebars.register_helper("map_generator", Box::new(map_generator_helper));

    handlebars.register_template_string("full", full_tpl).expect("Failed to register full template");
    handlebars.register_template_string("api", api_tpl).expect("Failed to register api template");

    let specs     = AMQProtocolDefinition::load();
    data.insert("specs".to_string(), specs);
    writeln!(f, "{}", handlebars.render("full", &data).expect("Failed to render full template"));
    writeln!(f2, "{}", handlebars.render("api", &data).expect("Failed to render full template"));

    //writeln!(f, "{}", specs.codegen_full(&full_tpl)).expect("Failed to generate generated.rs");
}

fn get_type_and_name(arg: &AMQPArgument, specs: &serde_json::Map<String, serde_json::Value>) -> (AMQPType, String) {
  match arg.amqp_type {
    Some(ref ty) => (ty.clone(), arg.name.to_string()),
    None     => {
      let domains: Vec<AMQPDomain> = serde_json::from_value(specs.get("domains").unwrap().clone()).unwrap();

      let arg_domain = arg.domain.as_ref().map(|d| d.to_string()).unwrap();
      let res = domains.iter().find(|d| d.name == arg_domain).map(|d| d.amqp_type.clone()).unwrap();
      //println!("key {} => domain gave type: {}", arg.name, res);
      (res, arg_domain.to_string())
    },
  }
}

fn map_type_helper(h: &Helper, _: &Handlebars, rc: &mut RenderContext) -> Result<(), RenderError> {
  println!("val1: {:?}", h.param(0));
  let val = h.param(0).unwrap().value().clone();
  let arg:AMQPArgument = serde_json::from_value(val.clone()).unwrap();

  let rendered = match arg.amqp_type {
    Some(ty) => ty.to_string(),
    None     => {
      let data:  serde_json::Map<String,  serde_json::Value> = serde_json::from_value(rc.context().data().clone()).unwrap();
      let specs: serde_json::Map<String, serde_json::Value> =  serde_json::from_value(data.get("specs").unwrap().clone()).unwrap();
      let domains: Vec<AMQPDomain> =  serde_json::from_value(specs.get("domains").unwrap().clone()).unwrap();

      let lookup_domain = arg.domain.clone().unwrap();
      let res = domains.iter().find(|d| d.name == lookup_domain).map(|d| d.amqp_type.to_string()).unwrap();
      println!("key {} => domain gave type: {}", arg.name, res);
      res
    },
  };
  try!(rc.writer.write(rendered.as_bytes()));

  Ok(())
}

fn map_parser_helper(h: &Helper, _: &Handlebars, rc: &mut RenderContext) -> Result<(), RenderError> {
  println!("val1: {:?}", h.param(0));
  let val = h.param(0).unwrap().value().clone();
  let arg:AMQPArgument = serde_json::from_value(val).unwrap();

  let rendered = match arg.amqp_type {
    Some(AMQPType::Boolean)        => "map!(be_u8, |u| u != 0)",
    Some(AMQPType::ShortShortInt)  => "be_i8",
    Some(AMQPType::ShortShortUInt) => "be_u8",
    Some(AMQPType::ShortInt)       => "be_i16",
    Some(AMQPType::ShortUInt)      => "be_u16",
    Some(AMQPType::LongInt)        => "be_i32",
    Some(AMQPType::LongUInt)       => "be_u32",
    Some(AMQPType::LongLongInt)    => "be_i64",
    Some(AMQPType::LongLongUInt)   => "be_u64",
    Some(AMQPType::ShortString)    => "map!(short_string, |s:&str| s.to_string())",
    Some(AMQPType::LongString)     => "map!(long_string,  |s:&str| s.to_string())",
    Some(AMQPType::FieldTable)     => "field_table",
    Some(AMQPType::Timestamp)      => "be_u64",
    Some(_)                        => unimplemented!(),
    None                           => {
      let data:  serde_json::Map<String, serde_json::Value> = serde_json::from_value(rc.context().data().clone()).unwrap();
      let specs: serde_json::Map<String, serde_json::Value> =  serde_json::from_value(data.get("specs").unwrap().clone()).unwrap();
      let domains: Vec<AMQPDomain> =  serde_json::from_value(specs.get("domains").unwrap().clone()).unwrap();

      let lookup_domain = arg.domain.clone().unwrap();
      domains.iter().find(|d| d.name == lookup_domain).map(|d| match d.amqp_type {
        AMQPType::Boolean        => "map!(be_u8, |u| u != 0)",
        AMQPType::ShortShortInt  => "be_i8",
        AMQPType::ShortShortUInt => "be_u8",
        AMQPType::ShortInt       => "be_i16",
        AMQPType::ShortUInt      => "be_u16",
        AMQPType::LongInt        => "be_i32",
        AMQPType::LongUInt       => "be_u32",
        AMQPType::LongLongInt    => "be_i64",
        AMQPType::LongLongUInt   => "be_u64",
        AMQPType::ShortString    => "map!(short_string, |s:&str| s.to_string())",
        AMQPType::LongString     => "map!(long_string,  |s:&str| s.to_string())",
        AMQPType::FieldTable     => "field_table",
        AMQPType::Timestamp      => "be_u64",
        _                        => unimplemented!(),
      }).unwrap()
    }
  };
  try!(rc.writer.write(rendered.as_bytes()));

  Ok(())
}

fn map_generator_helper(h: &Helper, _: &Handlebars, rc: &mut RenderContext) -> Result<(), RenderError> {
  println!("val1: {:?}", h.param(0));
  let val = h.param(0).unwrap().value().clone();
  let arg:AMQPArgument = serde_json::from_value(val).unwrap();

  let data:  serde_json::Map<String,  serde_json::Value> = serde_json::from_value(rc.context().data().clone()).unwrap();
  let specs: serde_json::Map<String, serde_json::Value> =  serde_json::from_value(data.get("specs").unwrap().clone()).unwrap();
  let (arg_type, arg_name) = get_type_and_name(&arg, &specs);

  let rendered = match arg_type {
    AMQPType::Boolean        => "gen_bool",
    AMQPType::ShortShortInt  => "gen_be_i8",
    AMQPType::ShortShortUInt => "gen_be_u8",
    AMQPType::ShortInt       => "gen_be_i16",
    AMQPType::ShortUInt      => "gen_be_u16",
    AMQPType::LongInt        => "gen_be_i32",
    AMQPType::LongUInt       => "gen_be_u32",
    AMQPType::LongLongInt    => "gen_be_i64",
    AMQPType::LongLongUInt   => "gen_be_u64",
    AMQPType::ShortString    => "gen_short_string",
    AMQPType::LongString     => "gen_long_string",
    AMQPType::FieldTable     => "gen_field_table",
    AMQPType::Timestamp      => "gen_be_u64",
    _                        => unimplemented!(),
  };
  try!(rc.writer.write(rendered.as_bytes()));

  Ok(())
}
<commit_msg>helper methods to access the specs<commit_after>extern crate amq_protocol_codegen;
extern crate amq_protocol_types;
extern crate handlebars;
extern crate serde_json;

use amq_protocol_codegen::*;
use amq_protocol_types::*;
use handlebars::{Handlebars,Helper,HelperDef,RenderContext,RenderError};

use std::collections::BTreeMap;
use std::env;
use std::fs::File;
use std::io::{Read, Write};
use std::path::Path;
use std::ascii::AsciiExt;

fn main() {
    let out_dir      = env::var("OUT_DIR").expect("OUT_DIR is not defined");
    let dest_path    = Path::new(&out_dir).join("generated.rs");
    let dest_path2   = Path::new(&out_dir).join("api.rs");
    let mut f        = File::create(&dest_path).expect("Failed to create generated.rs");
    let mut f2       = File::create(&dest_path2).expect("Failed to create api.rs");
    let mut full_tpl = String::new();
    let mut api_tpl = String::new();
    std::fs::File::open("templates/main.rs").expect("Failed to open main template").read_to_string(&mut full_tpl).expect("Failed to read main template");
    std::fs::File::open("templates/api.rs").expect("Failed to open main template").read_to_string(&mut api_tpl).expect("Failed to read main template");

    let mut handlebars = Handlebars::new();
    let mut data = BTreeMap::new();

    handlebars.register_escape_fn(handlebars::no_escape);
    handlebars.register_helper("snake", Box::new(snake_helper));
    handlebars.register_helper("camel", Box::new(camel_helper));
    handlebars.register_helper("map_type", Box::new(map_type_helper));
    handlebars.register_helper("map_parser", Box::new(map_parser_helper));
    handlebars.register_helper("map_generator", Box::new(map_generator_helper));

    handlebars.register_template_string("full", full_tpl).expect("Failed to register full template");
    handlebars.register_template_string("api", api_tpl).expect("Failed to register api template");

    let specs     = AMQProtocolDefinition::load();
    data.insert("specs".to_string(), specs);
    writeln!(f, "{}", handlebars.render("full", &data).expect("Failed to render full template"));
    writeln!(f2, "{}", handlebars.render("api", &data).expect("Failed to render full template"));

    //writeln!(f, "{}", specs.codegen_full(&full_tpl)).expect("Failed to generate generated.rs");
}

fn get_type_and_name(arg: &AMQPArgument, specs: &serde_json::Map<String, serde_json::Value>) -> (AMQPType, String) {
  match arg.amqp_type {
    Some(ref ty) => (ty.clone(), arg.name.to_string()),
    None     => {
      let domains: Vec<AMQPDomain> = serde_json::from_value(specs.get("domains").unwrap().clone()).unwrap();

      let arg_domain = arg.domain.as_ref().map(|d| d.to_string()).unwrap();
      let res = domains.iter().find(|d| d.name == arg_domain).map(|d| d.amqp_type.clone()).unwrap();
      //println!("key {} => domain gave type: {}", arg.name, res);
      (res, arg_domain.to_string())
    },
  }
}

fn map_type_helper(h: &Helper, _: &Handlebars, rc: &mut RenderContext) -> Result<(), RenderError> {
  println!("val1: {:?}", h.param(0));
  let val = h.param(0).unwrap().value().clone();
  let arg:AMQPArgument = serde_json::from_value(val.clone()).unwrap();

  let rendered = match arg.amqp_type {
    Some(ty) => ty.to_string(),
    None     => {
      let specs = get_specs(rc);
      let domains: Vec<AMQPDomain> = serde_json::from_value(specs.get("domains").unwrap().clone()).unwrap();

      let lookup_domain = arg.domain.clone().unwrap();
      let res = domains.iter().find(|d| d.name == lookup_domain).map(|d| d.amqp_type.to_string()).unwrap();
      println!("key {} => domain gave type: {}", arg.name, res);
      res
    },
  };
  try!(rc.writer.write(rendered.as_bytes()));

  Ok(())
}

fn get_specs(rc: &mut RenderContext) ->  serde_json::Map<String, serde_json::Value> {
  let data:  serde_json::Map<String, serde_json::Value> = serde_json::from_value(rc.context().data().clone()).unwrap();
  let specs: serde_json::Map<String, serde_json::Value> = serde_json::from_value(data.get("specs").unwrap().clone()).unwrap();

  specs
}

fn get_arguments(h: &Helper, rc: &mut RenderContext, class_id_index: usize, method_id_index: usize) -> Vec<AMQPArgument> {
  let class_id:  u8 = serde_json::from_value(h.param(class_id_index).unwrap().value().clone()).unwrap();
  let method_id: u8 = serde_json::from_value(h.param(method_id_index).unwrap().value().clone()).unwrap();

  let specs = get_specs(rc);

  let classes: Vec<AMQPClass> = serde_json::from_value(specs.get("classes").unwrap().clone()).unwrap();

  let arguments:Vec<AMQPArgument> = classes.iter().find(|c| c.id == class_id).and_then(|class| {
    class.methods.iter().find(|m| m.id == method_id)
  }).map(|m| m.arguments.clone()).unwrap();

  arguments
}
fn map_parser_helper(h: &Helper, _: &Handlebars, rc: &mut RenderContext) -> Result<(), RenderError> {
  println!("val1: {:?}", h.param(0));
  let val = h.param(0).unwrap().value().clone();
  let arg:AMQPArgument = serde_json::from_value(val).unwrap();

  let rendered = match arg.amqp_type {
    Some(AMQPType::Boolean)        => "map!(be_u8, |u| u != 0)",
    Some(AMQPType::ShortShortInt)  => "be_i8",
    Some(AMQPType::ShortShortUInt) => "be_u8",
    Some(AMQPType::ShortInt)       => "be_i16",
    Some(AMQPType::ShortUInt)      => "be_u16",
    Some(AMQPType::LongInt)        => "be_i32",
    Some(AMQPType::LongUInt)       => "be_u32",
    Some(AMQPType::LongLongInt)    => "be_i64",
    Some(AMQPType::LongLongUInt)   => "be_u64",
    Some(AMQPType::ShortString)    => "map!(short_string, |s:&str| s.to_string())",
    Some(AMQPType::LongString)     => "map!(long_string,  |s:&str| s.to_string())",
    Some(AMQPType::FieldTable)     => "field_table",
    Some(AMQPType::Timestamp)      => "be_u64",
    Some(_)                        => unimplemented!(),
    None                           => {
      let data:  serde_json::Map<String, serde_json::Value> = serde_json::from_value(rc.context().data().clone()).unwrap();
      let specs: serde_json::Map<String, serde_json::Value> =  serde_json::from_value(data.get("specs").unwrap().clone()).unwrap();
      let domains: Vec<AMQPDomain> =  serde_json::from_value(specs.get("domains").unwrap().clone()).unwrap();

      let lookup_domain = arg.domain.clone().unwrap();
      domains.iter().find(|d| d.name == lookup_domain).map(|d| match d.amqp_type {
        AMQPType::Boolean        => "map!(be_u8, |u| u != 0)",
        AMQPType::ShortShortInt  => "be_i8",
        AMQPType::ShortShortUInt => "be_u8",
        AMQPType::ShortInt       => "be_i16",
        AMQPType::ShortUInt      => "be_u16",
        AMQPType::LongInt        => "be_i32",
        AMQPType::LongUInt       => "be_u32",
        AMQPType::LongLongInt    => "be_i64",
        AMQPType::LongLongUInt   => "be_u64",
        AMQPType::ShortString    => "map!(short_string, |s:&str| s.to_string())",
        AMQPType::LongString     => "map!(long_string,  |s:&str| s.to_string())",
        AMQPType::FieldTable     => "field_table",
        AMQPType::Timestamp      => "be_u64",
        _                        => unimplemented!(),
      }).unwrap()
    }
  };
  try!(rc.writer.write(rendered.as_bytes()));

  Ok(())
}

fn map_generator_helper(h: &Helper, _: &Handlebars, rc: &mut RenderContext) -> Result<(), RenderError> {
  println!("val1: {:?}", h.param(0));
  let val = h.param(0).unwrap().value().clone();
  let arg:AMQPArgument = serde_json::from_value(val).unwrap();

  let specs = get_specs(rc);
  let (arg_type, arg_name) = get_type_and_name(&arg, &specs);

  let rendered = match arg_type {
    AMQPType::Boolean        => "gen_bool",
    AMQPType::ShortShortInt  => "gen_be_i8",
    AMQPType::ShortShortUInt => "gen_be_u8",
    AMQPType::ShortInt       => "gen_be_i16",
    AMQPType::ShortUInt      => "gen_be_u16",
    AMQPType::LongInt        => "gen_be_i32",
    AMQPType::LongUInt       => "gen_be_u32",
    AMQPType::LongLongInt    => "gen_be_i64",
    AMQPType::LongLongUInt   => "gen_be_u64",
    AMQPType::ShortString    => "gen_short_string",
    AMQPType::LongString     => "gen_long_string",
    AMQPType::FieldTable     => "gen_field_table",
    AMQPType::Timestamp      => "gen_be_u64",
    _                        => unimplemented!(),
  };
  try!(rc.writer.write(rendered.as_bytes()));

  Ok(())
}
<|endoftext|>


<commit_before>#![deny(warnings)]

extern crate cargo_registry;
extern crate chrono;
extern crate diesel;

use diesel::prelude::*;
use std::env;
use std::time::Duration;

use cargo_registry::models::VersionDownload;
use cargo_registry::schema::*;

static LIMIT: i64 = 1000;

fn main() {
    let daemon = env::args().nth(1).as_ref().map(|s| &s[..]) == Some("daemon");
    let sleep = env::args().nth(2).map(|s| s.parse().unwrap());
    loop {
        let conn = cargo_registry::db::connect_now().unwrap();
        update(&conn).unwrap();
        drop(conn);
        if daemon {
            std::thread::sleep(Duration::new(sleep.unwrap(), 0));
        } else {
            break;
        }
    }
}

fn update(conn: &PgConnection) -> QueryResult<()> {
    use version_downloads::dsl::*;
    let mut max = Some(0);
    while let Some(m) = max {
        conn.transaction::<_, diesel::result::Error, _>(|| {
            let rows = version_downloads
                .filter(processed.eq(false))
                .filter(id.gt(m))
                .order(id)
                .limit(LIMIT)
                .load(conn)?;
            collect(conn, &rows)?;
            max = rows.last().map(|d| d.id);
            Ok(())
        })?;
    }
    Ok(())
}

fn collect(conn: &PgConnection, rows: &[VersionDownload]) -> QueryResult<()> {
    use diesel::{insert_into, update};

    // Anything older than 24 hours ago will be frozen and will not be queried
    // against again.
    let now = chrono::Utc::now();
    let cutoff = now.naive_utc().date() - chrono::Duration::days(1);

    println!(
        "updating {} versions (cutoff {})",
        rows.len(),
        now.to_rfc2822()
    );

    let mut total = 0;
    for download in rows {
        let amt = download.downloads - download.counted;
        total += i64::from(amt);

        // Flag this row as having been processed if we're passed the cutoff,
        // and unconditionally increment the number of counted downloads.
        update(version_downloads::table.find(download.id))
            .set((
                version_downloads::processed.eq(download.date < cutoff),
                version_downloads::counted.eq(version_downloads::counted + amt),
            ))
            .execute(conn)?;

        // Update the total number of version downloads
        let crate_id = update(versions::table.find(download.version_id))
            .set(versions::downloads.eq(versions::downloads + amt))
            .returning(versions::crate_id)
            .get_result::<i32>(conn)?;

        // Update the total number of crate downloads
        update(crates::table.find(crate_id))
            .set(crates::downloads.eq(crates::downloads + amt))
            .execute(conn)?;

        // Update the total number of crate downloads for today
        insert_into(crate_downloads::table)
            .values((
                crate_downloads::crate_id.eq(crate_id),
                crate_downloads::downloads.eq(amt),
                crate_downloads::date.eq(download.date),
            ))
            .on_conflict(crate_downloads::table.primary_key())
            .do_update()
            .set(crate_downloads::downloads.eq(crate_downloads::downloads + amt))
            .execute(conn)?;
    }

    // After everything else is done, update the global counter of total
    // downloads.
    update(metadata::table)
        .set(metadata::total_downloads.eq(metadata::total_downloads + total))
        .execute(conn)?;

    Ok(())
}

#[cfg(test)]
mod test {
    extern crate semver;

    use std::collections::HashMap;

    use diesel::insert_into;
    use super::*;
    use cargo_registry::env;
    use cargo_registry::models::{Crate, NewCrate, NewUser, NewVersion, User, Version};

    fn conn() -> PgConnection {
        let conn = PgConnection::establish(&env("TEST_DATABASE_URL")).unwrap();
        conn.begin_test_transaction().unwrap();
        conn
    }

    fn user(conn: &PgConnection) -> User {
        NewUser::new(2, "login", None, None, None, "access_token")
            .create_or_update(conn)
            .unwrap()
    }

    fn crate_and_version(conn: &PgConnection, user_id: i32) -> (Crate, Version) {
        let krate = NewCrate {
            name: "foo",
            ..Default::default()
        }.create_or_update(&conn, None, user_id)
            .unwrap();
        let version = NewVersion::new(
            krate.id,
            &semver::Version::parse("1.0.0").unwrap(),
            &HashMap::new(),
            None,
            None,
        ).unwrap();
        let version = version.save(&conn, &[]).unwrap();
        (krate, version)
    }

    macro_rules! crate_downloads {
        ($conn: expr, $id: expr, $expected: expr) => {
            let dl = crate_downloads::table
                .filter(crate_downloads::crate_id.eq($id))
                .select(crate_downloads::downloads)
                .first($conn);
            assert_eq!(Ok($expected as i32), dl);
        }
    }

    #[test]
    fn increment() {
        use diesel::dsl::*;

        let conn = conn();
        let user = user(&conn);
        let (krate, version) = crate_and_version(&conn, user.id);
        insert_into(version_downloads::table)
            .values(version_downloads::version_id.eq(version.id))
            .execute(&conn)
            .unwrap();
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::date.eq(date(now - 1.day())),
                version_downloads::processed.eq(true),
            ))
            .execute(&conn)
            .unwrap();

        ::update(&conn).unwrap();
        let version_downloads = versions::table
            .find(version.id)
            .select(versions::downloads)
            .first(&conn);
        assert_eq!(Ok(1), version_downloads);
        let crate_downloads = crates::table
            .find(krate.id)
            .select(crates::downloads)
            .first(&conn);
        assert_eq!(Ok(1), crate_downloads);
        crate_downloads!(&conn, krate.id, 1);
        ::update(&conn).unwrap();
        let version_downloads = versions::table
            .find(version.id)
            .select(versions::downloads)
            .first(&conn);
        assert_eq!(Ok(1), version_downloads);
    }

    #[test]
    fn set_processed_true() {
        use diesel::dsl::*;

        let conn = conn();
        let user = user(&conn);
        let (_, version) = crate_and_version(&conn, user.id);
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::downloads.eq(2),
                version_downloads::counted.eq(2),
                version_downloads::date.eq(date(now - 2.days())),
                version_downloads::processed.eq(false),
            ))
            .execute(&conn)
            .unwrap();
        ::update(&conn).unwrap();
        let processed = version_downloads::table
            .filter(version_downloads::version_id.eq(version.id))
            .select(version_downloads::processed)
            .first(&conn);
        assert_eq!(Ok(true), processed);
    }

    #[test]
    fn dont_process_recent_row() {
        use diesel::dsl::*;
        let conn = conn();
        let user = user(&conn);
        let (_, version) = crate_and_version(&conn, user.id);
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::downloads.eq(2),
                version_downloads::counted.eq(2),
                version_downloads::date.eq(date(now - 2.hours())),
                version_downloads::processed.eq(false),
            ))
            .execute(&conn)
            .unwrap();
        ::update(&conn).unwrap();
        let processed = version_downloads::table
            .filter(version_downloads::version_id.eq(version.id))
            .select(version_downloads::processed)
            .first(&conn);
        assert_eq!(Ok(false), processed);
    }

    #[test]
    fn increment_a_little() {
        use diesel::dsl::*;
        use diesel::update;

        let conn = conn();
        let user = user(&conn);
        let (krate, version) = crate_and_version(&conn, user.id);
        update(versions::table)
            .set(versions::updated_at.eq(now - 2.hours()))
            .execute(&conn)
            .unwrap();
        update(crates::table)
            .set(crates::updated_at.eq(now - 2.hours()))
            .execute(&conn)
            .unwrap();
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::downloads.eq(2),
                version_downloads::counted.eq(1),
                version_downloads::date.eq(date(now)),
                version_downloads::processed.eq(false),
            ))
            .execute(&conn)
            .unwrap();
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::date.eq(date(now - 1.day())),
            ))
            .execute(&conn)
            .unwrap();

        let version_before = versions::table
            .find(version.id)
            .first::<Version>(&conn)
            .unwrap();
        let krate_before = Crate::all()
            .filter(crates::id.eq(krate.id))
            .first::<Crate>(&conn)
            .unwrap();
        ::update(&conn).unwrap();
        let version2 = versions::table
            .find(version.id)
            .first::<Version>(&conn)
            .unwrap();
        assert_eq!(version2.downloads, 2);
        assert_eq!(version2.updated_at, version_before.updated_at);
        let krate2 = Crate::all()
            .filter(crates::id.eq(krate.id))
            .first::<Crate>(&conn)
            .unwrap();
        assert_eq!(krate2.downloads, 2);
        assert_eq!(krate2.updated_at, krate_before.updated_at);
        crate_downloads!(&conn, krate.id, 1);
        ::update(&conn).unwrap();
        let version3 = versions::table
            .find(version.id)
            .first::<Version>(&conn)
            .unwrap();
        assert_eq!(version3.downloads, 2);
    }

    #[test]
    fn set_processed_no_set_updated_at() {
        use diesel::update;
        use diesel::dsl::*;

        let conn = conn();
        let user = user(&conn);
        let (_, version) = crate_and_version(&conn, user.id);
        update(versions::table)
            .set(versions::updated_at.eq(now - 2.days()))
            .execute(&conn)
            .unwrap();
        update(crates::table)
            .set(crates::updated_at.eq(now - 2.days()))
            .execute(&conn)
            .unwrap();
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::downloads.eq(2),
                version_downloads::counted.eq(2),
                version_downloads::date.eq(date(now - 2.days())),
                version_downloads::processed.eq(false),
            ))
            .execute(&conn)
            .unwrap();

        ::update(&conn).unwrap();
        let versions_changed = versions::table
            .select(versions::updated_at.ne(now - 2.days()))
            .get_result(&conn);
        let crates_changed = crates::table
            .select(crates::updated_at.ne(now - 2.days()))
            .get_result(&conn);
        assert_eq!(Ok(false), versions_changed);
        assert_eq!(Ok(false), crates_changed);
    }
}
<commit_msg>Don't lock more than 1 row at a time in `update-downloads`<commit_after>#![deny(warnings)]

extern crate cargo_registry;
extern crate chrono;
extern crate diesel;

use diesel::prelude::*;
use std::env;
use std::time::Duration;

use cargo_registry::models::VersionDownload;
use cargo_registry::schema::*;

static LIMIT: i64 = 1000;

fn main() {
    let daemon = env::args().nth(1).as_ref().map(|s| &s[..]) == Some("daemon");
    let sleep = env::args().nth(2).map(|s| s.parse().unwrap());
    loop {
        let conn = cargo_registry::db::connect_now().unwrap();
        update(&conn).unwrap();
        drop(conn);
        if daemon {
            std::thread::sleep(Duration::new(sleep.unwrap(), 0));
        } else {
            break;
        }
    }
}

fn update(conn: &PgConnection) -> QueryResult<()> {
    use version_downloads::dsl::*;

    let mut max = Some(0);
    while let Some(m) = max {
        let rows = version_downloads
            .filter(processed.eq(false))
            .filter(id.gt(m))
            .order(id)
            .limit(LIMIT)
            .load(conn)?;
        collect(conn, &rows)?;
        max = rows.last().map(|d| d.id);
    }
    Ok(())
}

fn collect(conn: &PgConnection, rows: &[VersionDownload]) -> QueryResult<()> {
    use diesel::{insert_into, update};

    // Anything older than 24 hours ago will be frozen and will not be queried
    // against again.
    let now = chrono::Utc::now();
    let cutoff = now.naive_utc().date() - chrono::Duration::days(1);

    println!(
        "updating {} versions (cutoff {})",
        rows.len(),
        now.to_rfc2822()
    );

    for download in rows {
        let amt = download.downloads - download.counted;

        conn.transaction::<_, diesel::result::Error, _>(|| {
            // Flag this row as having been processed if we're passed the cutoff,
            // and unconditionally increment the number of counted downloads.
            update(version_downloads::table.find(download.id))
                .set((
                    version_downloads::processed.eq(download.date < cutoff),
                    version_downloads::counted.eq(version_downloads::counted + amt),
                ))
                .execute(conn)?;

            // Update the total number of version downloads
            let crate_id = update(versions::table.find(download.version_id))
                .set(versions::downloads.eq(versions::downloads + amt))
                .returning(versions::crate_id)
                .get_result::<i32>(conn)?;

            // Update the total number of crate downloads
            update(crates::table.find(crate_id))
                .set(crates::downloads.eq(crates::downloads + amt))
                .execute(conn)?;

            // Update the total number of crate downloads for today
            insert_into(crate_downloads::table)
                .values((
                    crate_downloads::crate_id.eq(crate_id),
                    crate_downloads::downloads.eq(amt),
                    crate_downloads::date.eq(download.date),
                ))
                .on_conflict(crate_downloads::table.primary_key())
                .do_update()
                .set(crate_downloads::downloads.eq(crate_downloads::downloads + amt))
                .execute(conn)?;

            // Now that everything else for this crate is done, update the global counter of total
            // downloads
            update(metadata::table)
                .set(metadata::total_downloads.eq(metadata::total_downloads + (amt as i64)))
                .execute(conn)?;

            Ok(())
        })?;
    }

    Ok(())
}

#[cfg(test)]
mod test {
    extern crate semver;

    use std::collections::HashMap;

    use diesel::insert_into;
    use super::*;
    use cargo_registry::env;
    use cargo_registry::models::{Crate, NewCrate, NewUser, NewVersion, User, Version};

    fn conn() -> PgConnection {
        let conn = PgConnection::establish(&env("TEST_DATABASE_URL")).unwrap();
        conn.begin_test_transaction().unwrap();
        conn
    }

    fn user(conn: &PgConnection) -> User {
        NewUser::new(2, "login", None, None, None, "access_token")
            .create_or_update(conn)
            .unwrap()
    }

    fn crate_and_version(conn: &PgConnection, user_id: i32) -> (Crate, Version) {
        let krate = NewCrate {
            name: "foo",
            ..Default::default()
        }.create_or_update(&conn, None, user_id)
            .unwrap();
        let version = NewVersion::new(
            krate.id,
            &semver::Version::parse("1.0.0").unwrap(),
            &HashMap::new(),
            None,
            None,
        ).unwrap();
        let version = version.save(&conn, &[]).unwrap();
        (krate, version)
    }

    macro_rules! crate_downloads {
        ($conn: expr, $id: expr, $expected: expr) => {
            let dl = crate_downloads::table
                .filter(crate_downloads::crate_id.eq($id))
                .select(crate_downloads::downloads)
                .first($conn);
            assert_eq!(Ok($expected as i32), dl);
        }
    }

    #[test]
    fn increment() {
        use diesel::dsl::*;

        let conn = conn();
        let user = user(&conn);
        let (krate, version) = crate_and_version(&conn, user.id);
        insert_into(version_downloads::table)
            .values(version_downloads::version_id.eq(version.id))
            .execute(&conn)
            .unwrap();
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::date.eq(date(now - 1.day())),
                version_downloads::processed.eq(true),
            ))
            .execute(&conn)
            .unwrap();

        ::update(&conn).unwrap();
        let version_downloads = versions::table
            .find(version.id)
            .select(versions::downloads)
            .first(&conn);
        assert_eq!(Ok(1), version_downloads);
        let crate_downloads = crates::table
            .find(krate.id)
            .select(crates::downloads)
            .first(&conn);
        assert_eq!(Ok(1), crate_downloads);
        crate_downloads!(&conn, krate.id, 1);
        ::update(&conn).unwrap();
        let version_downloads = versions::table
            .find(version.id)
            .select(versions::downloads)
            .first(&conn);
        assert_eq!(Ok(1), version_downloads);
    }

    #[test]
    fn set_processed_true() {
        use diesel::dsl::*;

        let conn = conn();
        let user = user(&conn);
        let (_, version) = crate_and_version(&conn, user.id);
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::downloads.eq(2),
                version_downloads::counted.eq(2),
                version_downloads::date.eq(date(now - 2.days())),
                version_downloads::processed.eq(false),
            ))
            .execute(&conn)
            .unwrap();
        ::update(&conn).unwrap();
        let processed = version_downloads::table
            .filter(version_downloads::version_id.eq(version.id))
            .select(version_downloads::processed)
            .first(&conn);
        assert_eq!(Ok(true), processed);
    }

    #[test]
    fn dont_process_recent_row() {
        use diesel::dsl::*;
        let conn = conn();
        let user = user(&conn);
        let (_, version) = crate_and_version(&conn, user.id);
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::downloads.eq(2),
                version_downloads::counted.eq(2),
                version_downloads::date.eq(date(now - 2.hours())),
                version_downloads::processed.eq(false),
            ))
            .execute(&conn)
            .unwrap();
        ::update(&conn).unwrap();
        let processed = version_downloads::table
            .filter(version_downloads::version_id.eq(version.id))
            .select(version_downloads::processed)
            .first(&conn);
        assert_eq!(Ok(false), processed);
    }

    #[test]
    fn increment_a_little() {
        use diesel::dsl::*;
        use diesel::update;

        let conn = conn();
        let user = user(&conn);
        let (krate, version) = crate_and_version(&conn, user.id);
        update(versions::table)
            .set(versions::updated_at.eq(now - 2.hours()))
            .execute(&conn)
            .unwrap();
        update(crates::table)
            .set(crates::updated_at.eq(now - 2.hours()))
            .execute(&conn)
            .unwrap();
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::downloads.eq(2),
                version_downloads::counted.eq(1),
                version_downloads::date.eq(date(now)),
                version_downloads::processed.eq(false),
            ))
            .execute(&conn)
            .unwrap();
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::date.eq(date(now - 1.day())),
            ))
            .execute(&conn)
            .unwrap();

        let version_before = versions::table
            .find(version.id)
            .first::<Version>(&conn)
            .unwrap();
        let krate_before = Crate::all()
            .filter(crates::id.eq(krate.id))
            .first::<Crate>(&conn)
            .unwrap();
        ::update(&conn).unwrap();
        let version2 = versions::table
            .find(version.id)
            .first::<Version>(&conn)
            .unwrap();
        assert_eq!(version2.downloads, 2);
        assert_eq!(version2.updated_at, version_before.updated_at);
        let krate2 = Crate::all()
            .filter(crates::id.eq(krate.id))
            .first::<Crate>(&conn)
            .unwrap();
        assert_eq!(krate2.downloads, 2);
        assert_eq!(krate2.updated_at, krate_before.updated_at);
        crate_downloads!(&conn, krate.id, 1);
        ::update(&conn).unwrap();
        let version3 = versions::table
            .find(version.id)
            .first::<Version>(&conn)
            .unwrap();
        assert_eq!(version3.downloads, 2);
    }

    #[test]
    fn set_processed_no_set_updated_at() {
        use diesel::update;
        use diesel::dsl::*;

        let conn = conn();
        let user = user(&conn);
        let (_, version) = crate_and_version(&conn, user.id);
        update(versions::table)
            .set(versions::updated_at.eq(now - 2.days()))
            .execute(&conn)
            .unwrap();
        update(crates::table)
            .set(crates::updated_at.eq(now - 2.days()))
            .execute(&conn)
            .unwrap();
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::downloads.eq(2),
                version_downloads::counted.eq(2),
                version_downloads::date.eq(date(now - 2.days())),
                version_downloads::processed.eq(false),
            ))
            .execute(&conn)
            .unwrap();

        ::update(&conn).unwrap();
        let versions_changed = versions::table
            .select(versions::updated_at.ne(now - 2.days()))
            .get_result(&conn);
        let crates_changed = crates::table
            .select(crates::updated_at.ne(now - 2.days()))
            .get_result(&conn);
        assert_eq!(Ok(false), versions_changed);
        assert_eq!(Ok(false), crates_changed);
    }
}
<|endoftext|>


<commit_before>use std::ffi::OsStr;
use std::fs;
use std::path;

use liquid;
use serde_json;
use serde_yaml;
use toml;

use error::*;

use super::files;

#[derive(Debug, Clone, PartialEq, Default)]
#[derive(Serialize, Deserialize)]
#[serde(deny_unknown_fields, default)]
pub struct SiteBuilder {
    pub title: Option<String>,
    pub description: Option<String>,
    pub base_url: Option<String>,
    pub data: Option<liquid::Object>,
    pub data_dir: Option<path::PathBuf>,
}

impl SiteBuilder {
    pub fn build(self) -> Result<liquid::Object> {
        let SiteBuilder {
            title,
            description,
            base_url,
            data,
            data_dir,
        } = self;

        let base_url = base_url.map(|mut l| {
                                        if l.ends_with('/') {
                                            l.pop();
                                        }
                                        l
                                    });

        let mut attributes = liquid::Object::new();
        if let Some(title) = title {
            attributes.insert("title".to_owned(), liquid::Value::scalar(title));
        }
        if let Some(description) = description {
            attributes.insert("description".to_owned(), liquid::Value::scalar(description));
        }
        if let Some(base_url) = base_url {
            attributes.insert("base_url".to_owned(), liquid::Value::scalar(base_url));
        }
        let mut data = data.unwrap_or_default();
        if let Some(ref data_dir) = data_dir {
            insert_data_dir(&mut data, &data_dir)?;
        }
        if !data.is_empty() {
            attributes.insert("data".to_owned(), liquid::Value::Object(data));
        }

        Ok(attributes)
    }
}

fn deep_insert(data_map: &mut liquid::Object,
               file_path: &path::Path,
               target_key: String,
               data: liquid::Value)
               -> Result<()> {
    // now find the nested map it is supposed to be in
    let target_map = if let Some(path) = file_path.parent() {
        let mut map = data_map;
        for part in path.iter() {
            let key = part.to_str().ok_or_else(|| {
                format!("The data from {:?} can't be loaded as it contains non utf-8 characters",
                        path)
            })?;
            let cur_map = map;
            map = cur_map
                .entry(String::from(key))
                .or_insert_with(|| liquid::Value::Object(liquid::Object::new()))
                .as_object_mut()
                .ok_or_else(|| {
                                format!("Aborting: Duplicate in data tree. Would overwrite {:?} ",
                                        path)
                            })?;
        }
        map
    } else {
        data_map
    };

    match target_map.insert(target_key, data) {
        None => Ok(()),
        _ => {
            Err(format!("The data from {:?} can't be loaded: the key already exists",
                        file_path)
                    .into())
        }
    }
}
fn load_data(data_path: &path::Path) -> Result<liquid::Value> {
    let ext = data_path.extension().unwrap_or_else(|| OsStr::new(""));

    let data: liquid::Value;

    if ext == OsStr::new("yml") || ext == OsStr::new("yaml") {
        let reader = fs::File::open(data_path)?;
        data = serde_yaml::from_reader(reader)?;
    } else if ext == OsStr::new("json") {
        let reader = fs::File::open(data_path)?;
        data = serde_json::from_reader(reader)?;
    } else if ext == OsStr::new("toml") {
        let text = files::read_file(data_path)?;
        data = toml::from_str(&text)?;
    } else {
        bail!("Failed to load of data {:?}: unknown file type '{:?}'.\n\
              Supported data files extensions are: yml, yaml, json and toml.",
              data_path,
              ext);
    }

    Ok(data)
}

fn insert_data_dir(data: &mut liquid::Object, data_root: &path::Path) -> Result<()> {
    debug!("Loading data from {:?}", data_root);

    let data_files_builder = files::FilesBuilder::new(data_root)?;
    let data_files = data_files_builder.build()?;
    for full_path in data_files.files() {
        let rel_path = full_path
            .strip_prefix(data_root)
            .expect("file was found under the root");

        let file_stem = full_path
            .file_stem()
            .expect("Files will always return with a stem");
        let file_stem = String::from(file_stem.to_str().unwrap());
        let data_fragment = load_data(&full_path)
            .chain_err(|| format!("Loading data from {:?} failed", full_path))?;

        deep_insert(data, rel_path, file_stem, data_fragment)
            .chain_err(|| format!("Merging data into {:?} failed", rel_path))?;
    }

    Ok(())
}
<commit_msg>style: Missing separator<commit_after>use std::ffi::OsStr;
use std::fs;
use std::path;

use liquid;
use serde_json;
use serde_yaml;
use toml;

use error::*;

use super::files;

#[derive(Debug, Clone, PartialEq, Default)]
#[derive(Serialize, Deserialize)]
#[serde(deny_unknown_fields, default)]
pub struct SiteBuilder {
    pub title: Option<String>,
    pub description: Option<String>,
    pub base_url: Option<String>,
    pub data: Option<liquid::Object>,
    pub data_dir: Option<path::PathBuf>,
}

impl SiteBuilder {
    pub fn build(self) -> Result<liquid::Object> {
        let SiteBuilder {
            title,
            description,
            base_url,
            data,
            data_dir,
        } = self;

        let base_url = base_url.map(|mut l| {
                                        if l.ends_with('/') {
                                            l.pop();
                                        }
                                        l
                                    });

        let mut attributes = liquid::Object::new();
        if let Some(title) = title {
            attributes.insert("title".to_owned(), liquid::Value::scalar(title));
        }
        if let Some(description) = description {
            attributes.insert("description".to_owned(), liquid::Value::scalar(description));
        }
        if let Some(base_url) = base_url {
            attributes.insert("base_url".to_owned(), liquid::Value::scalar(base_url));
        }
        let mut data = data.unwrap_or_default();
        if let Some(ref data_dir) = data_dir {
            insert_data_dir(&mut data, &data_dir)?;
        }
        if !data.is_empty() {
            attributes.insert("data".to_owned(), liquid::Value::Object(data));
        }

        Ok(attributes)
    }
}

fn deep_insert(data_map: &mut liquid::Object,
               file_path: &path::Path,
               target_key: String,
               data: liquid::Value)
               -> Result<()> {
    // now find the nested map it is supposed to be in
    let target_map = if let Some(path) = file_path.parent() {
        let mut map = data_map;
        for part in path.iter() {
            let key = part.to_str().ok_or_else(|| {
                format!("The data from {:?} can't be loaded as it contains non utf-8 characters",
                        path)
            })?;
            let cur_map = map;
            map = cur_map
                .entry(String::from(key))
                .or_insert_with(|| liquid::Value::Object(liquid::Object::new()))
                .as_object_mut()
                .ok_or_else(|| {
                                format!("Aborting: Duplicate in data tree. Would overwrite {:?} ",
                                        path)
                            })?;
        }
        map
    } else {
        data_map
    };

    match target_map.insert(target_key, data) {
        None => Ok(()),
        _ => {
            Err(format!("The data from {:?} can't be loaded: the key already exists",
                        file_path)
                    .into())
        }
    }
}

fn load_data(data_path: &path::Path) -> Result<liquid::Value> {
    let ext = data_path.extension().unwrap_or_else(|| OsStr::new(""));

    let data: liquid::Value;

    if ext == OsStr::new("yml") || ext == OsStr::new("yaml") {
        let reader = fs::File::open(data_path)?;
        data = serde_yaml::from_reader(reader)?;
    } else if ext == OsStr::new("json") {
        let reader = fs::File::open(data_path)?;
        data = serde_json::from_reader(reader)?;
    } else if ext == OsStr::new("toml") {
        let text = files::read_file(data_path)?;
        data = toml::from_str(&text)?;
    } else {
        bail!("Failed to load of data {:?}: unknown file type '{:?}'.\n\
              Supported data files extensions are: yml, yaml, json and toml.",
              data_path,
              ext);
    }

    Ok(data)
}

fn insert_data_dir(data: &mut liquid::Object, data_root: &path::Path) -> Result<()> {
    debug!("Loading data from {:?}", data_root);

    let data_files_builder = files::FilesBuilder::new(data_root)?;
    let data_files = data_files_builder.build()?;
    for full_path in data_files.files() {
        let rel_path = full_path
            .strip_prefix(data_root)
            .expect("file was found under the root");

        let file_stem = full_path
            .file_stem()
            .expect("Files will always return with a stem");
        let file_stem = String::from(file_stem.to_str().unwrap());
        let data_fragment = load_data(&full_path)
            .chain_err(|| format!("Loading data from {:?} failed", full_path))?;

        deep_insert(data, rel_path, file_stem, data_fragment)
            .chain_err(|| format!("Merging data into {:?} failed", rel_path))?;
    }

    Ok(())
}
<|endoftext|>