<commit_before>// Copyright 2020-2021 The Propverify authors
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use log::info;
use regex::Regex;
use std::ffi::OsStr;
use std::path::PathBuf;
use structopt::StructOpt;

use inkwell::context::Context;
use inkwell::memory_buffer::MemoryBuffer;
use inkwell::module::Linkage;
use inkwell::module::Module;
use inkwell::types::{AnyType, FunctionType};
use inkwell::values::{AnyValue, BasicValue, BasicValueEnum};
use inkwell::values::{FunctionValue, GlobalValue, PointerValue};
use inkwell::AddressSpace;

// Command line argument parsing
#[derive(StructOpt)]
#[structopt(
    name = "rvt-patch-llvm",
    about = "Preprocess rustc generated llvm code for verification.",
    // version number is taken automatically from Cargo.toml
)]
struct Opt {
    /// Input file.
    #[structopt(name = "INPUT", parse(from_os_str))]
    input: PathBuf,

    /// Output file
    #[structopt(
        short,
        long,
        name = "OUTPUT",
        parse(from_os_str),
        default_value = "out"
    )]
    output: PathBuf,

    /// Call initializers from main
    #[structopt(short, long)]
    initializers: bool,

    /// SeaHorn preparation (conflicts with --initializers)
    #[structopt(short, long, conflicts_with = "initializers")]
    seahorn: bool,

    /// Select a specific test to run (instead of 'main')
    #[structopt(short, long, name = "TEST")]
    test: Option<String>,

    /// Increase message verbosity
    #[structopt(short, long, parse(from_occurrences))]
    verbosity: usize,
}

fn main() {
    let opt = Opt::from_args();

    #[rustfmt::skip]
    stderrlog::new()
        .verbosity(opt.verbosity)
        .init()
        .unwrap();

    // Read the input file
    info!("Reading input from {}", opt.input.to_str().unwrap());
    let memory_buffer =
        MemoryBuffer::create_from_file(&opt.input).expect("ERROR: failed to open file.");

    let context = Context::create();
    let mut module = context
        .create_module_from_ir(memory_buffer)
        .expect("ERROR: failed to create module.");

    if opt.initializers {
        handle_initializers(&context, &mut module);
    }

    if opt.seahorn {
        handle_main(&module, &opt.test);

        handle_panic(&module);

        replace_def_with_dec(
            &module,
            &Regex::new(r"^_ZN3std2io5stdio7_eprint17h[a-f0-9]{16}E$").unwrap(),
        );
        replace_def_with_dec(
            &module,
            &Regex::new(r"^_ZN3std2io5stdio6_print17h[a-f0-9]{16}E$").unwrap(),
        );
    }

    // Write output file
    info!("Writing output to {}", opt.output.to_str().unwrap());
    if opt.output.extension() == Some(OsStr::new("bc")) {
        // output bitcode
        // TODO: this function returns bool but the doc doesn't say anything about it.
        module.write_bitcode_to_path(&opt.output);
    } else {
        // output disassembled bitcode
        module
            .print_to_file(&opt.output)
            .expect("ERROR: failed to write to file.");
    }
}

////////////////////////////////////////////////////////////////
// Transformations associated with initializers
////////////////////////////////////////////////////////////////

fn handle_initializers(context: &Context, module: &mut Module) {
    if let Some(initializer) =
        collect_initializers(context, module, ".init_array", "__init_function")
    {
        info!(
            "Combined .init_array* initializers into '{}'",
            initializer.get_name().to_str().unwrap()
        );

        let main = module
            .get_function("main")
            .expect("Unable to find 'main' function");
        let mut args = get_fn_args(main);
        assert!(args.len() == 2); // We expect "i32 @main(i32 %0, i8** %1)"
        let i8_type = context.i8_type();
        let pi8_type = i8_type.ptr_type(AddressSpace::Generic);
        let ppi8_type = pi8_type.ptr_type(AddressSpace::Generic);
        args.push(ppi8_type.const_null().as_basic_value_enum());
        insert_call_at_head(context, initializer, args, main);
        info!(
            "Inserted call to '{}' into 'main'",
            initializer.get_name().to_str().unwrap()
        )
    } else {
        info!("No initializers to handle")
    }
}

/// Collect all the initializers in a section (whose name starts with 'prefix')
/// into a single function that calls all the initializers.
fn collect_initializers<'a>(
    context: &Context,
    module: &mut Module<'a>,
    prefix: &str,
    nm: &str,
) -> Option<FunctionValue<'a>> {
    let vs = collect_variables_in_section(module, prefix);
    for v in &vs {
        info!("Found initializer {:?}", v.get_name().to_str().unwrap());
    }

    let fps: Vec<PointerValue> = vs.iter().map(get_initializer_function).collect();

    if !fps.is_empty() {
        let fp = fps[0];

        // dereference the pointer type
        let fp_type = fp.get_type().get_element_type().into_function_type();
        info!("Initializer type {:?}", fp_type.print_to_string());

        Some(build_fanout(context, module, nm, fp_type, fps))
    } else {
        None
    }
}

/// Collect variables that are assigned to a section whose name matches prefix
fn collect_variables_in_section<'a>(module: &Module<'a>, prefix: &str) -> Vec<GlobalValue<'a>> {
    let mut vs = Vec::new();
    // todo: should implement an iterator for global values so that the following loop
    // becomes just iter().filter_map().filter_map().filter()
    let mut og = module.get_first_global();
    while let Some(g) = og {
        if let Some(s) = g.get_section() {
            if let Ok(s) = s.to_str() {
                if s.starts_with(prefix) {
                    vs.push(g)
                }
            }
        }
        og = g.get_next_global();
    }
    vs
}

/// Convert the contents of an initializer section to a function pointer.
///
/// Initializer sections contain structs where the first field is a function pointer cast to
/// some other type.
fn get_initializer_function<'a>(v: &GlobalValue<'a>) -> PointerValue<'a> {
    let i = v.get_initializer().unwrap().into_struct_value();
    assert!(i.get_num_operands() == 2); // expecting two fields in struct
    let i = i.get_operand(0).unwrap();
    assert!(i.get_num_operands() == 1); // expecting bitcast
    let fp = i.get_operand(0).unwrap();
    fp.into_pointer_value()
}

/// Given a list of functions of type 'ty', build a function that calls each function in order
///
/// Assumes (without checking) that return type is void
///
///     define void @fanout(i32 %0, i32 %1) {
///     entry:
///       call void @f1(i32 %0, i32 %1)
///       call void @f2(i32 %0, i32 %1)
///       call void @f3(i32 %0, i32 %1)
///       ret void
///     }
///
fn build_fanout<'a>(
    context: &Context,
    module: &mut Module<'a>,
    nm: &str,
    ty: FunctionType<'a>,
    fps: Vec<PointerValue<'a>>,
) -> FunctionValue<'a> {
    let function = module.add_function(nm, ty, None);
    let args = get_fn_args(function);
    let basic_block = context.append_basic_block(function, "entry");
    let builder = context.create_builder();
    builder.position_at_end(basic_block);

    for fp in fps {
        builder.build_call(fp, &args, "");
        builder.build_return(None);
        // println!("Built function {:?}", function)
    }

    function
}

fn get_fn_args<'a>(function: FunctionValue<'a>) -> Vec<BasicValueEnum<'a>> {
    (0..function.count_params())
        .map(|i| function.get_nth_param(i).unwrap())
        .collect()
}

fn insert_call_at_head<'a>(
    context: &Context,
    f: FunctionValue<'a>,
    args: Vec<BasicValueEnum<'a>>,
    insertee: FunctionValue<'a>,
) {
    let bb = insertee
        .get_first_basic_block()
        .expect("Unable to find function to insert function call into");
    let first_instruction = bb
        .get_first_instruction()
        .expect("Unable to find where to insert function call into function");
    let builder = context.create_builder();
    builder.position_before(&first_instruction);
    builder.build_call(f, &args, "");
}

////////////////////////////////////////////////////////////////
// Transformations associated with SeaHorn
////////////////////////////////////////////////////////////////

fn handle_main(module: &Module, otest: &Option<String>) {
    // Remove the main function rustc generates.
    if let Some(main) = module.get_function("main") {
        unsafe {
            // Any use of `main` after calling `delete` is unsafe
            main.delete();
        }
        info!("Deleted 'main' (was added by rustc).");
    }

    let re = match otest {
        Some(test) => {
            Regex::new(&format!("{}{}{}", test.len(), &test, r"17h[a-f0-9]{16}E$")).unwrap()
        }
        None => Regex::new(r"4main17h[a-f0-9]{16}E$").unwrap(),
    };

    // Print the entry point function name
    if let Some(fun) = get_function(module, &re) {
        // Change the linkage of mangled main function from internal to external.
        // fun.set_linkage(Linkage::External);
        println!("ENTRY: {}", fun.get_name().to_str().unwrap());
    }
}

fn get_function<'ctx>(module: &'ctx Module, re: &Regex) -> Option<FunctionValue<'ctx>> {
    let mut op_fun = module.get_first_function();
    while let Some(fun) = op_fun {
        if re.is_match(
            fun.get_name()
                .to_str()
                .expect("ERROR: function name is not in valid UTF-8"),
        ) {
            return Some(fun);
        }
        op_fun = fun.get_next_function();
    }
    None
}

fn handle_panic(module: &Module) {
    // TODO: make "spanic" a CL arg.
    if let Some(spanic) = module.get_function("spanic") {
        if let Some(unwind) = module.get_function("rust_begin_unwind") {
            unwind.replace_all_uses_with(spanic);
            info!("Replaced panic handling ('rust_begin_unwind') with 'spanic'.");
        }
    }
}

fn replace_def_with_dec(module: &Module, re: &Regex) {
    if let Some(fun) = get_function(module, re) {
        for bb in fun.get_basic_blocks() {
            unsafe {
                // Any use of `bb` after calling `delete` is unsafe
                bb.delete().unwrap();
            }
        }
        fun.remove_personality_function();
        fun.set_linkage(Linkage::External);
        info!(
            "Removed the implementation of '{}'.",
            fun.get_name().to_str().unwrap()
        );
    }
}

////////////////////////////////////////////////////////////////
// End
////////////////////////////////////////////////////////////////
<commit_msg>RVT-Patch-LLVM: Add --feature<commit_after>// Copyright 2020-2021 The Propverify authors
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use log::info;
use regex::Regex;
use std::ffi::OsStr;
use std::path::PathBuf;
use structopt::StructOpt;

use inkwell::builder::Builder;
use inkwell::context::Context;
use inkwell::memory_buffer::MemoryBuffer;
use inkwell::module::Linkage;
use inkwell::module::Module;
use inkwell::types::{AnyType, FunctionType};
use inkwell::values::{AnyValue, BasicValue, BasicValueEnum};
use inkwell::values::{FunctionValue, GlobalValue, PointerValue};
use inkwell::AddressSpace;

// Command line argument parsing
#[derive(StructOpt)]
#[structopt(
    name = "rvt-patch-llvm",
    about = "Preprocess rustc generated llvm code for verification.",
    // version number is taken automatically from Cargo.toml
)]
struct Opt {
    /// Input file.
    #[structopt(name = "INPUT", parse(from_os_str))]
    input: PathBuf,

    /// Output file
    #[structopt(
        short,
        long,
        name = "OUTPUT",
        parse(from_os_str),
        default_value = "out"
    )]
    output: PathBuf,

    /// Eliminate feature tests
    #[structopt(short, long)]
    features: bool,

    /// Call initializers from main
    #[structopt(short, long)]
    initializers: bool,

    /// SeaHorn preparation (conflicts with --initializers)
    #[structopt(short, long, conflicts_with = "initializers")]
    seahorn: bool,

    /// Select a specific test to run (instead of 'main')
    #[structopt(short, long, name = "TEST")]
    test: Option<String>,

    /// Increase message verbosity
    #[structopt(short, long, parse(from_occurrences))]
    verbosity: usize,
}

fn main() {
    let opt = Opt::from_args();

    #[rustfmt::skip]
    stderrlog::new()
        .verbosity(opt.verbosity)
        .init()
        .unwrap();

    // Read the input file
    info!("Reading input from {}", opt.input.to_str().unwrap());
    let memory_buffer =
        MemoryBuffer::create_from_file(&opt.input).expect("ERROR: failed to open file.");

    let context = Context::create();
    let mut module = context
        .create_module_from_ir(memory_buffer)
        .expect("ERROR: failed to create module.");

    if opt.features {
        info!("Patching feature functions");
        stub_functions(
            &context,
            &module,
            &[
                "std::std_detect::detect::arch::__is_feature_detected::avx2",
                "std::std_detect::detect::arch::__is_feature_detected::ssse3",
                "std::std_detect::detect::check_for",
            ],
            |builder| {
                builder.build_return(Some(&context.bool_type().const_int(0, false)));
                ()
            },
        );
    }

    if opt.initializers {
        handle_initializers(&context, &mut module);
    }

    if opt.seahorn {
        handle_main(&module, &opt.test);

        handle_panic(&module);

        replace_def_with_dec(
            &module,
            &Regex::new(r"^_ZN3std2io5stdio7_eprint17h[a-f0-9]{16}E$").unwrap(),
        );
        replace_def_with_dec(
            &module,
            &Regex::new(r"^_ZN3std2io5stdio6_print17h[a-f0-9]{16}E$").unwrap(),
        );
    }

    // Write output file
    info!("Writing output to {}", opt.output.to_str().unwrap());
    if opt.output.extension() == Some(OsStr::new("bc")) {
        // output bitcode
        // TODO: this function returns bool but the doc doesn't say anything about it.
        module.write_bitcode_to_path(&opt.output);
    } else {
        // output disassembled bitcode
        module
            .print_to_file(&opt.output)
            .expect("ERROR: failed to write to file.");
    }
}

////////////////////////////////////////////////////////////////
// Transformations associated with initializers
////////////////////////////////////////////////////////////////

fn handle_initializers(context: &Context, module: &mut Module) {
    if let Some(initializer) =
        collect_initializers(context, module, ".init_array", "__init_function")
    {
        info!(
            "Combined .init_array* initializers into '{}'",
            initializer.get_name().to_str().unwrap()
        );

        let main = module
            .get_function("main")
            .expect("Unable to find 'main' function");
        let mut args = get_fn_args(main);
        assert!(args.len() == 2); // We expect "i32 @main(i32 %0, i8** %1)"
        let i8_type = context.i8_type();
        let pi8_type = i8_type.ptr_type(AddressSpace::Generic);
        let ppi8_type = pi8_type.ptr_type(AddressSpace::Generic);
        args.push(ppi8_type.const_null().as_basic_value_enum());
        insert_call_at_head(context, initializer, args, main);
        info!(
            "Inserted call to '{}' into 'main'",
            initializer.get_name().to_str().unwrap()
        )
    } else {
        info!("No initializers to handle")
    }
}

/// Collect all the initializers in a section (whose name starts with 'prefix')
/// into a single function that calls all the initializers.
fn collect_initializers<'a>(
    context: &Context,
    module: &mut Module<'a>,
    prefix: &str,
    nm: &str,
) -> Option<FunctionValue<'a>> {
    let vs = collect_variables_in_section(module, prefix);
    for v in &vs {
        info!("Found initializer {:?}", v.get_name().to_str().unwrap());
    }

    let fps: Vec<PointerValue> = vs.iter().map(get_initializer_function).collect();

    if !fps.is_empty() {
        let fp = fps[0];

        // dereference the pointer type
        let fp_type = fp.get_type().get_element_type().into_function_type();
        info!("Initializer type {:?}", fp_type.print_to_string());

        Some(build_fanout(context, module, nm, fp_type, fps))
    } else {
        None
    }
}

/// Collect variables that are assigned to a section whose name matches prefix
fn collect_variables_in_section<'a>(module: &Module<'a>, prefix: &str) -> Vec<GlobalValue<'a>> {
    let mut vs = Vec::new();
    // todo: should implement an iterator for global values so that the following loop
    // becomes just iter().filter_map().filter_map().filter()
    let mut og = module.get_first_global();
    while let Some(g) = og {
        if let Some(s) = g.get_section() {
            if let Ok(s) = s.to_str() {
                if s.starts_with(prefix) {
                    vs.push(g)
                }
            }
        }
        og = g.get_next_global();
    }
    vs
}

/// Convert the contents of an initializer section to a function pointer.
///
/// Initializer sections contain structs where the first field is a function pointer cast to
/// some other type.
fn get_initializer_function<'a>(v: &GlobalValue<'a>) -> PointerValue<'a> {
    let i = v.get_initializer().unwrap().into_struct_value();
    assert!(i.get_num_operands() == 2); // expecting two fields in struct
    let i = i.get_operand(0).unwrap();
    assert!(i.get_num_operands() == 1); // expecting bitcast
    let fp = i.get_operand(0).unwrap();
    fp.into_pointer_value()
}

/// Given a list of functions of type 'ty', build a function that calls each function in order
///
/// Assumes (without checking) that return type is void
///
///     define void @fanout(i32 %0, i32 %1) {
///     entry:
///       call void @f1(i32 %0, i32 %1)
///       call void @f2(i32 %0, i32 %1)
///       call void @f3(i32 %0, i32 %1)
///       ret void
///     }
///
fn build_fanout<'a>(
    context: &Context,
    module: &mut Module<'a>,
    nm: &str,
    ty: FunctionType<'a>,
    fps: Vec<PointerValue<'a>>,
) -> FunctionValue<'a> {
    let function = module.add_function(nm, ty, None);
    let args = get_fn_args(function);
    let basic_block = context.append_basic_block(function, "entry");
    let builder = context.create_builder();
    builder.position_at_end(basic_block);

    for fp in fps {
        builder.build_call(fp, &args, "");
        builder.build_return(None);
        // println!("Built function {:?}", function)
    }

    function
}

fn get_fn_args<'a>(function: FunctionValue<'a>) -> Vec<BasicValueEnum<'a>> {
    (0..function.count_params())
        .map(|i| function.get_nth_param(i).unwrap())
        .collect()
}

fn insert_call_at_head<'a>(
    context: &Context,
    f: FunctionValue<'a>,
    args: Vec<BasicValueEnum<'a>>,
    insertee: FunctionValue<'a>,
) {
    let bb = insertee
        .get_first_basic_block()
        .expect("Unable to find function to insert function call into");
    let first_instruction = bb
        .get_first_instruction()
        .expect("Unable to find where to insert function call into function");
    let builder = context.create_builder();
    builder.position_before(&first_instruction);
    builder.build_call(f, &args, "");
}

////////////////////////////////////////////////////////////////
// Function transformation functions
////////////////////////////////////////////////////////////////

/// Apply Rust mangling rule to a function name like 'foo::bar'.
/// Does not insert a unique hash at the end or the final 'E' character.
fn rust_mangle(rust_name: &str) -> String {
    let mangled =
        &rust_name
        .split("::")
        .into_iter()
        .map(|x| format!("{}{}", x.len(), x))
        .collect::<Vec<String>>()
        .concat();
    "_ZN".to_string() + mangled
}

/// Delete the body of a function
fn delete_body<'ctx>(fun: &FunctionValue<'ctx>) {
    for bb in fun.get_basic_blocks() {
        unsafe {
            // Any use of `bb` after calling `delete` is unsafe
            bb.delete().expect("delete basic block");
        }
    }
}

/// Stub out a list of functions by deleting the existing
/// body of each function and using mk_stub to construct a new body.
///
/// Functions are identified specified by names like "foo::bar"
/// and exclude the uniquifying hash code at the end.
fn stub_functions<F>(context: &Context, module: &Module, functions: &[&str], mk_stub: F)
where
    F: Fn(&Builder),
{
    for f in functions {
        if let Some(fun) = get_function_by_unmangled_name(&module, f) {
            info!("Stubbing out function function {}", f);
            delete_body(&fun);
            let basic_block = context.append_basic_block(fun, "entry");
            let builder = context.create_builder();
            builder.position_at_end(basic_block);
            mk_stub(&builder)
        } else {
            info!("Did not find function to stub {}", f)
        }
    }
}

////////////////////////////////////////////////////////////////
// Transformations associated with SeaHorn
////////////////////////////////////////////////////////////////

fn handle_main(module: &Module, otest: &Option<String>) {
    // Remove the main function rustc generates.
    if let Some(main) = module.get_function("main") {
        unsafe {
            // Any use of `main` after calling `delete` is unsafe
            main.delete();
        }
        info!("Deleted 'main' (was added by rustc).");
    }

    let re = match otest {
        Some(test) => {
            Regex::new(&format!("{}{}{}", test.len(), &test, r"17h[a-f0-9]{16}E$")).unwrap()
        }
        None => Regex::new(r"4main17h[a-f0-9]{16}E$").unwrap(),
    };

    // Print the entry point function name
    if let Some(fun) = get_function_by_regex(module, &re) {
        // Change the linkage of mangled main function from internal to external.
        // fun.set_linkage(Linkage::External);
        println!("ENTRY: {}", fun.get_name().to_str().unwrap());
    }
}

/// Find a function whose name matches regex `re`
fn get_function_by_regex<'ctx>(module: &'ctx Module, re: &Regex) -> Option<FunctionValue<'ctx>> {
    get_function(module, |name| re.is_match(name))
}

/// Find a function whose name matches a Rust-mangled prefix
fn get_function_by_unmangled_name<'ctx>(module: &'ctx Module, prefix: &str) -> Option<FunctionValue<'ctx>> {
    let prefix = rust_mangle(prefix);
    get_function(module, |name| name.starts_with(&prefix))
}

/// Find a function whose name satisfies predicate `is_match`
fn get_function<'ctx, P>(module: &'ctx Module, is_match: P) -> Option<FunctionValue<'ctx>>
where
    P: Fn(&str) -> bool,
{
    let mut op_fun = module.get_first_function();
    while let Some(fun) = op_fun {
        if is_match(
            fun.get_name()
                .to_str()
                .expect("ERROR: function name is not in valid UTF-8"),
        ) {
            return Some(fun);
        }
        op_fun = fun.get_next_function();
    }
    None
}

fn handle_panic(module: &Module) {
    // TODO: make "spanic" a CL arg.
    if let Some(spanic) = module.get_function("spanic") {
        if let Some(unwind) = module.get_function("rust_begin_unwind") {
            unwind.replace_all_uses_with(spanic);
            info!("Replaced panic handling ('rust_begin_unwind') with 'spanic'.");
        }
    }
}

/// Change a function to a declaration by
/// deleting all basic blocks and modifying metadata
/// such as personality_function, linkage, etc.
fn replace_def_with_dec(module: &Module, re: &Regex) {
    if let Some(fun) = get_function_by_regex(module, re) {
        delete_body(&fun);
        fun.remove_personality_function();
        fun.set_linkage(Linkage::External);
        info!(
            "Removed the implementation of '{}'.",
            fun.get_name().to_str().unwrap()
        );
    }
}

////////////////////////////////////////////////////////////////
// End
////////////////////////////////////////////////////////////////
<|endoftext|>


<commit_before>use super::*;

use super::contest::ContestView;
use crate::api::award::AwardOutcome;
use root::ApiContext;

use diesel::{QueryDsl, RunQueryDsl};
use file::FileContentInput;
use juniper::FieldResult;
use problem::material::Material;
use problem::ProblemName;

use crate::data::file::FileContent;
use schema::problems;
use std::path::PathBuf;
use user::UserId;

#[derive(juniper::GraphQLInputObject, Insertable)]
#[table_name = "problems"]
pub struct ProblemInput {
    name: String,
    archive_content: FileContentInput,
}

#[derive(juniper::GraphQLInputObject, AsChangeset)]
#[table_name = "problems"]
pub struct ProblemUpdateInput {
    name: String,
    archive_content: Option<FileContentInput>,
}

#[derive(Queryable, Clone, Debug)]
struct ProblemData {
    name: String,
    archive_content: FileContent,
}

/// A problem in the contest
pub struct Problem<'a> {
    context: &'a ApiContext<'a>,
    data: ProblemData,
}

#[juniper_ext::graphql]
impl Problem<'_> {
    /// Name of this problem. Unique in the current contest.
    fn name(&self) -> ProblemName {
        ProblemName(self.data.name.clone())
    }

    /// Material of this problem
    fn material(&self) -> FieldResult<Material> {
        Ok(self.get_problem_material()?)
    }
}

impl Problem<'_> {
    /// Get a problem data by its name
    pub fn by_name<'a>(context: &'a ApiContext<'a>, name: &str) -> FieldResult<Problem<'a>> {
        let data = problems::table.find(name).first(&context.database)?;
        Ok(Problem { context, data })
    }

    /// Get all the problems data in the database
    pub fn all<'a>(context: &'a ApiContext<'a>) -> FieldResult<Vec<Problem>> {
        Ok(problems::table
            .load::<ProblemData>(&context.database)?
            .into_iter()
            .map(|data| Problem { context, data })
            .collect())
    }

    /// Insert a problem in the database
    pub fn insert(context: &ApiContext, inputs: Vec<ProblemInput>) -> FieldResult<()> {
        diesel::insert_into(problems::table)
            .values(inputs)
            .execute(&context.database)?;
        Ok(())
    }

    /// Delete a problem from the database
    pub fn delete(context: &ApiContext, names: Vec<String>) -> FieldResult<()> {
        use crate::diesel::ExpressionMethods;
        diesel::delete(problems::table)
            .filter(problems::dsl::name.eq_any(names))
            .execute(&context.database)?;
        Ok(())
    }

    /// Update a problem in the database
    pub fn update(context: &ApiContext, inputs: Vec<ProblemUpdateInput>) -> FieldResult<()> {
        use crate::diesel::ExpressionMethods;
        for input in inputs {
            diesel::update(problems::table)
                .filter(problems::dsl::name.eq(&input.name))
                .set(&input)
                .execute(&context.database)?;
        }
        Ok(())
    }

    pub fn unpack(&self) -> PathBuf {
        self.context
            .unpack_archive(&self.data.archive_content.0, "problem")
    }

    /// Material of this problem
    fn get_problem_material(&self) -> FieldResult<Material> {
        Ok(task_maker::generate_material(self.unpack())?)
    }
}

/// A problem in the contest as seen by contestants
pub struct ProblemView<'a> {
    contest_view: &'a ContestView<'a>,
    problem: Problem<'a>,
}

/// A problem in a contest
#[juniper_ext::graphql]
#[allow(dead_code)]
impl ProblemView<'_> {
    /// Name of this problem. Unique in the current contest.
    fn name(&self) -> ProblemName {
        self.problem.name()
    }

    /// Material of this problem
    fn material(&self) -> FieldResult<Material> {
        self.problem.material()
    }

    /// Material of this problem
    fn tackling(&self) -> Option<ProblemTackling> {
        if self.contest_view.user_id().is_some() {
            Some(ProblemTackling { problem: &self })
        } else {
            None
        }
    }
}

impl<'a> ProblemView<'a> {
    pub fn by_name(contest_view: &'a ContestView<'a>, name: &str) -> FieldResult<Self> {
        let problem = Problem::by_name(contest_view.context(), name)?;
        Ok(ProblemView {
            contest_view,
            problem,
        })
    }

    /// Get all the problems data in the database
    pub fn all(contest_view: &'a ContestView<'a>) -> FieldResult<Vec<Self>> {
        let problems = Problem::all(contest_view.context())?;
        Ok(problems
            .into_iter()
            .map(|problem| ProblemView {
                contest_view,
                problem,
            })
            .collect())
    }
}

/// Attempts at solving a problem by a user in the contest
pub struct ProblemTackling<'a> {
    /// The problem
    problem: &'a ProblemView<'a>,
}

impl ProblemTackling<'_> {
    fn user_id(&self) -> UserId {
        self.problem.contest_view.user_id().clone().unwrap()
    }

    fn name(&self) -> String {
        self.problem.name().0
    }
}

/// Attempts at solving a problem by a user in the contest
#[juniper_ext::graphql]
#[allow(dead_code)]
impl ProblemTackling<'_> {
    /// Score awards of the current user (if to be shown)
    fn awards(&self) -> FieldResult<Vec<AwardOutcome>> {
        Ok(AwardOutcome::by_user_and_problem(
            &self.problem.contest_view.context(),
            &self.user_id().0,
            &self.name(),
        )?)
    }

    /// Submissions of the current user (if to be shown)
    fn submissions(&self) -> FieldResult<Vec<contest_submission::Submission>> {
        Ok(contest_submission::Submission::by_user_and_problem(
            &self.problem.contest_view.context(),
            &self.user_id().0,
            &self.name(),
        )?)
    }

    /// Indicates if the user can submit to this problem
    fn can_submit(&self) -> bool {
        true
    }
}
<commit_msg>Cache problem material<commit_after>use std::path::PathBuf;

use diesel::{QueryDsl, RunQueryDsl};
use juniper::FieldResult;

use file::FileContentInput;
use problem::material::Material;
use problem::ProblemName;
use root::ApiContext;
use schema::problems;
use user::UserId;

use crate::api::award::AwardOutcome;
use crate::data::file::FileContent;

use super::contest::ContestView;
use super::*;

#[derive(juniper::GraphQLInputObject, Insertable)]
#[table_name = "problems"]
pub struct ProblemInput {
    name: String,
    archive_content: FileContentInput,
}

#[derive(juniper::GraphQLInputObject, AsChangeset)]
#[table_name = "problems"]
pub struct ProblemUpdateInput {
    name: String,
    archive_content: Option<FileContentInput>,
}

#[derive(Queryable, Clone, Debug)]
struct ProblemData {
    name: String,
    archive_content: FileContent,
}

/// A problem in the contest
pub struct Problem<'a> {
    context: &'a ApiContext<'a>,
    data: ProblemData,
    material: Material,
}

#[juniper_ext::graphql]
impl Problem<'_> {
    /// Name of this problem. Unique in the current contest.
    fn name(&self) -> ProblemName {
        ProblemName(self.data.name.clone())
    }

    /// Material of this problem
    fn material(&self) -> &Material {
        &self.material
    }
}

impl<'a> Problem<'a> {
    fn new(context: &'a ApiContext<'a>, data: ProblemData) -> FieldResult<Self> {
        let material = Self::get_problem_material(&data, context)?;
        Ok(Problem {
            context,
            data,
            material,
        })
    }

    /// Get a problem data by its name
    pub fn by_name(context: &'a ApiContext<'a>, name: &str) -> FieldResult<Self> {
        let data = problems::table.find(name).first(&context.database)?;
        Ok(Self::new(context, data)?)
    }

    /// Get all the problems data in the database
    pub fn all(context: &'a ApiContext<'a>) -> FieldResult<Vec<Self>> {
        Ok(problems::table
            .load::<ProblemData>(&context.database)?
            .into_iter()
            .map(|data| Self::new(context, data))
            .collect::<Result<Vec<_>, _>>()?)
    }

    /// Insert a problem in the database
    pub fn insert(context: &ApiContext, inputs: Vec<ProblemInput>) -> FieldResult<()> {
        diesel::insert_into(problems::table)
            .values(inputs)
            .execute(&context.database)?;
        Ok(())
    }

    /// Delete a problem from the database
    pub fn delete(context: &ApiContext, names: Vec<String>) -> FieldResult<()> {
        use crate::diesel::ExpressionMethods;
        diesel::delete(problems::table)
            .filter(problems::dsl::name.eq_any(names))
            .execute(&context.database)?;
        Ok(())
    }

    /// Update a problem in the database
    pub fn update(context: &ApiContext, inputs: Vec<ProblemUpdateInput>) -> FieldResult<()> {
        use crate::diesel::ExpressionMethods;
        for input in inputs {
            diesel::update(problems::table)
                .filter(problems::dsl::name.eq(&input.name))
                .set(&input)
                .execute(&context.database)?;
        }
        Ok(())
    }

    pub fn unpack(&self) -> PathBuf {
        Self::unpack_data(&self.data, &self.context)
    }

    fn unpack_data(data: &ProblemData, context: &ApiContext) -> PathBuf {
        context.unpack_archive(&data.archive_content.0, "problem")
    }

    /// Material of this problem
    fn get_problem_material(data: &ProblemData, context: &ApiContext) -> FieldResult<Material> {
        Ok(task_maker::generate_material(Self::unpack_data(
            data, context,
        ))?)
    }
}

/// A problem in the contest as seen by contestants
pub struct ProblemView<'a> {
    contest_view: &'a ContestView<'a>,
    problem: Problem<'a>,
}

/// A problem in a contest
#[juniper_ext::graphql]
#[allow(dead_code)]
impl ProblemView<'_> {
    /// Name of this problem. Unique in the current contest.
    fn name(&self) -> ProblemName {
        self.problem.name()
    }

    /// Material of this problem
    fn material(&self) -> &Material {
        self.problem.material()
    }

    /// Material of this problem
    fn tackling(&self) -> Option<ProblemTackling> {
        if self.contest_view.user_id().is_some() {
            Some(ProblemTackling { problem: &self })
        } else {
            None
        }
    }
}

impl<'a> ProblemView<'a> {
    pub fn by_name(contest_view: &'a ContestView<'a>, name: &str) -> FieldResult<Self> {
        let problem = Problem::by_name(contest_view.context(), name)?;
        Ok(ProblemView {
            contest_view,
            problem,
        })
    }

    /// Get all the problems data in the database
    pub fn all(contest_view: &'a ContestView<'a>) -> FieldResult<Vec<Self>> {
        let problems = Problem::all(contest_view.context())?;
        Ok(problems
            .into_iter()
            .map(|problem| ProblemView {
                contest_view,
                problem,
            })
            .collect())
    }
}

/// Attempts at solving a problem by a user in the contest
pub struct ProblemTackling<'a> {
    /// The problem
    problem: &'a ProblemView<'a>,
}

impl ProblemTackling<'_> {
    fn user_id(&self) -> UserId {
        self.problem.contest_view.user_id().clone().unwrap()
    }

    fn name(&self) -> String {
        self.problem.name().0
    }
}

/// Attempts at solving a problem by a user in the contest
#[juniper_ext::graphql]
#[allow(dead_code)]
impl ProblemTackling<'_> {
    /// Score awards of the current user (if to be shown)
    fn awards(&self) -> FieldResult<Vec<AwardOutcome>> {
        Ok(AwardOutcome::by_user_and_problem(
            &self.problem.contest_view.context(),
            &self.user_id().0,
            &self.name(),
        )?)
    }

    /// Submissions of the current user (if to be shown)
    fn submissions(&self) -> FieldResult<Vec<contest_submission::Submission>> {
        Ok(contest_submission::Submission::by_user_and_problem(
            &self.problem.contest_view.context(),
            &self.user_id().0,
            &self.name(),
        )?)
    }

    /// Indicates if the user can submit to this problem
    fn can_submit(&self) -> bool {
        true
    }
}
<|endoftext|>


<commit_before>#![feature(plugin)]
#![crate_type="staticlib"]
#![no_std]

//! Sample application for BlueNRG communication over SPI in X-NUCLEO-IDB04A1
//! extension board for NUCLEO-L152RE

#[phase(plugin, link)]
extern crate core;
extern crate zinc;

use core::intrinsics::abort;
// "curious module hack" - a workaround for `fmt` module being looked in `std`
// by the `write!` macro
mod std {
  pub use core::fmt;
}

//TODO(kvark): temporary `u8 -> str` conversion until #235 is resolved
fn map_byte(s: u8) -> (&'static str, &'static str) {
  fn map_hex(h: u8) -> &'static str {
      match h {
        0x0 => "0",
        0x1 => "1",
        0x2 => "2",
        0x3 => "3",
        0x4 => "4",
        0x5 => "5",
        0x6 => "6",
        0x7 => "7",
        0x8 => "8",
        0x9 => "9",
        0xA => "A",
        0xB => "B",
        0xC => "C",
        0xD => "D",
        0xE => "E",
        0xF => "F",
        _ => "",
      }
  }
  (map_hex(s>>4), map_hex(s&0xF))
}

#[no_mangle]
pub unsafe fn main() {
  use core::fmt::Writer;
  use core::result::Result;
  use zinc::drivers::bluenrg;
  use zinc::hal;
  use zinc::hal::pin::Gpio;
  use zinc::hal::stm32l1::{init, pin, spi, usart};

  zinc::hal::mem_init::init_stack();
  zinc::hal::mem_init::init_data();

  let sys_clock = init::ClockConfig::new_default();
  sys_clock.setup();

  let _usart_tx = pin::Pin::new(pin::Port::PortA, 2,
    pin::Mode::AltFunction(
      pin::AltMode::AfUsart1_Usart2_Usart3,
      pin::OutputType::OutPushPull,
      pin::Speed::VeryLow),
    pin::PullType::PullNone);

  let mut uart = usart::Usart::new(usart::UsartPeripheral::Usart2, 38400, usart::WordLen::WordLen8bits,
    hal::uart::Parity::Disabled, usart::StopBit::StopBit1bit, &sys_clock);
  let _ = write!(&mut uart, "BlueNRG test app for STM32L1\n");

  let _spi_clock = pin::Pin::new(pin::Port::PortB, 3,
    pin::Mode::AltFunction(pin::AltMode::AfSpi1_Spi2, pin::OutputType::OutPushPull, pin::Speed::Medium),
    pin::PullType::PullDown);

  let _spi_in = pin::Pin::new(pin::Port::PortA, 6,
    pin::Mode::AltFunction(pin::AltMode::AfSpi1_Spi2, pin::OutputType::OutPushPull, pin::Speed::Medium),
    pin::PullType::PullNone);

  let _spi_out = pin::Pin::new(pin::Port::PortA, 7,
    pin::Mode::AltFunction(pin::AltMode::AfSpi1_Spi2, pin::OutputType::OutPushPull, pin::Speed::Medium),
    pin::PullType::PullNone);

  let spi_csn = pin::Pin::new(pin::Port::PortA, 1,
    pin::Mode::GpioOut(pin::OutputType::OutPushPull, pin::Speed::Medium),
    pin::PullType::PullUp);
  spi_csn.set_high();

  let spi = spi::Spi::new(spi::Peripheral::Spi1, spi::Direction::FullDuplex,
    spi::Role::Master, spi::DataSize::U8, spi::DataFormat::MsbFirst, 1).
    unwrap_or_else(|_| {
      let _ = write!(&mut uart, "SPI failed to initialize");
      abort()
    });

  let bnrg_reset = pin::Pin::new(pin::Port::PortA, 8,
    pin::Mode::GpioOut(pin::OutputType::OutPushPull, pin::Speed::VeryLow),
    pin::PullType::PullUp);

  bnrg_reset.set_low();
  let _ = write!(&mut uart, "SPI created, status = {}\n",
    map_byte(spi.get_status()));
  bnrg_reset.set_high();

  let blue = bluenrg::BlueNrg::new(spi_csn, spi);

  let _ = match blue.wakeup(100) {
    Result::Ok((size_write, size_read)) => write!(&mut uart,
      "BlueNRG is ready, write size = {}, read size = {}\n",
      map_byte(size_write as u8), map_byte(size_read as u8)),
    Result::Err(bluenrg::Error::Sleeping) => write!(&mut uart,
      "BlueNRG is sleeping\n"),
    Result::Err(bluenrg::Error::Allocating) => write!(&mut uart,
      "BlueNRG is allocating buffers\n"),
    Result::Err(bluenrg::Error::Unknown(status)) => write!(&mut uart,
      "BlueNRG unknown status = {}\n", map_byte(status)),
    Result::Err(bluenrg::Error::BufferSize(_)) => write!(&mut uart, ""),
  };

  loop {}
}
<commit_msg>bluenrg: macro_use instead of phase(plugin)<commit_after>#![feature(plugin)]
#![crate_type="staticlib"]
#![no_std]

//! Sample application for BlueNRG communication over SPI in X-NUCLEO-IDB04A1
//! extension board for NUCLEO-L152RE

#[macro_use] #[plugin]
extern crate core;
extern crate zinc;

use core::intrinsics::abort;
// "curious module hack" - a workaround for `fmt` module being looked in `std`
// by the `write!` macro
mod std {
  pub use core::fmt;
}

//TODO(kvark): temporary `u8 -> str` conversion until #235 is resolved
fn map_byte(s: u8) -> (&'static str, &'static str) {
  fn map_hex(h: u8) -> &'static str {
      match h {
        0x0 => "0",
        0x1 => "1",
        0x2 => "2",
        0x3 => "3",
        0x4 => "4",
        0x5 => "5",
        0x6 => "6",
        0x7 => "7",
        0x8 => "8",
        0x9 => "9",
        0xA => "A",
        0xB => "B",
        0xC => "C",
        0xD => "D",
        0xE => "E",
        0xF => "F",
        _ => "",
      }
  }
  (map_hex(s>>4), map_hex(s&0xF))
}

#[no_mangle]
pub unsafe fn main() {
  use core::fmt::Writer;
  use core::result::Result;
  use zinc::drivers::bluenrg;
  use zinc::hal;
  use zinc::hal::pin::Gpio;
  use zinc::hal::stm32l1::{init, pin, spi, usart};

  zinc::hal::mem_init::init_stack();
  zinc::hal::mem_init::init_data();

  let sys_clock = init::ClockConfig::new_default();
  sys_clock.setup();

  let _usart_tx = pin::Pin::new(pin::Port::PortA, 2,
    pin::Mode::AltFunction(
      pin::AltMode::AfUsart1_Usart2_Usart3,
      pin::OutputType::OutPushPull,
      pin::Speed::VeryLow),
    pin::PullType::PullNone);

  let mut uart = usart::Usart::new(usart::UsartPeripheral::Usart2, 38400, usart::WordLen::WordLen8bits,
    hal::uart::Parity::Disabled, usart::StopBit::StopBit1bit, &sys_clock);
  let _ = write!(&mut uart, "BlueNRG test app for STM32L1\n");

  let _spi_clock = pin::Pin::new(pin::Port::PortB, 3,
    pin::Mode::AltFunction(pin::AltMode::AfSpi1_Spi2, pin::OutputType::OutPushPull, pin::Speed::Medium),
    pin::PullType::PullDown);

  let _spi_in = pin::Pin::new(pin::Port::PortA, 6,
    pin::Mode::AltFunction(pin::AltMode::AfSpi1_Spi2, pin::OutputType::OutPushPull, pin::Speed::Medium),
    pin::PullType::PullNone);

  let _spi_out = pin::Pin::new(pin::Port::PortA, 7,
    pin::Mode::AltFunction(pin::AltMode::AfSpi1_Spi2, pin::OutputType::OutPushPull, pin::Speed::Medium),
    pin::PullType::PullNone);

  let spi_csn = pin::Pin::new(pin::Port::PortA, 1,
    pin::Mode::GpioOut(pin::OutputType::OutPushPull, pin::Speed::Medium),
    pin::PullType::PullUp);
  spi_csn.set_high();

  let spi = spi::Spi::new(spi::Peripheral::Spi1, spi::Direction::FullDuplex,
    spi::Role::Master, spi::DataSize::U8, spi::DataFormat::MsbFirst, 1).
    unwrap_or_else(|_| {
      let _ = write!(&mut uart, "SPI failed to initialize");
      abort()
    });

  let bnrg_reset = pin::Pin::new(pin::Port::PortA, 8,
    pin::Mode::GpioOut(pin::OutputType::OutPushPull, pin::Speed::VeryLow),
    pin::PullType::PullUp);

  bnrg_reset.set_low();
  let _ = write!(&mut uart, "SPI created, status = {}\n",
    map_byte(spi.get_status()));
  bnrg_reset.set_high();

  let blue = bluenrg::BlueNrg::new(spi_csn, spi);

  let _ = match blue.wakeup(100) {
    Result::Ok((size_write, size_read)) => write!(&mut uart,
      "BlueNRG is ready, write size = {}, read size = {}\n",
      map_byte(size_write as u8), map_byte(size_read as u8)),
    Result::Err(bluenrg::Error::Sleeping) => write!(&mut uart,
      "BlueNRG is sleeping\n"),
    Result::Err(bluenrg::Error::Allocating) => write!(&mut uart,
      "BlueNRG is allocating buffers\n"),
    Result::Err(bluenrg::Error::Unknown(status)) => write!(&mut uart,
      "BlueNRG unknown status = {}\n", map_byte(status)),
    Result::Err(bluenrg::Error::BufferSize(_)) => write!(&mut uart, ""),
  };

  loop {}
}
<|endoftext|>


<commit_before>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use serde::{Deserialize, Serialize};

// The maximum message size per state sync message
pub const MAX_MESSAGE_SIZE: usize = 12 * 1024 * 1024; /* 12 MiB */

#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StateSyncConfig {
    pub data_streaming_service: DataStreamingServiceConfig,
    pub aptos_data_client: AptosDataClientConfig,
    pub state_sync_driver: StateSyncDriverConfig,
    pub storage_service: StorageServiceConfig,
}

/// The bootstrapping mode determines how the node will bootstrap to the latest
/// blockchain state, e.g., directly download the latest states.
#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
pub enum BootstrappingMode {
    ApplyTransactionOutputsFromGenesis, // Applies transaction outputs (starting at genesis)
    DownloadLatestStates, // Downloads the state keys and values (at the latest version)
    ExecuteTransactionsFromGenesis, // Executes transactions (starting at genesis)
}

impl BootstrappingMode {
    pub fn to_label(&self) -> &'static str {
        match self {
            BootstrappingMode::ApplyTransactionOutputsFromGenesis => {
                "apply_transaction_outputs_from_genesis"
            }
            BootstrappingMode::DownloadLatestStates => "download_latest_states",
            BootstrappingMode::ExecuteTransactionsFromGenesis => {
                "execute_transactions_from_genesis"
            }
        }
    }
}

/// The continuous syncing mode determines how the node will stay up-to-date
/// once it has bootstrapped and the blockchain continues to grow, e.g.,
/// continuously executing all transactions.
#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
pub enum ContinuousSyncingMode {
    ApplyTransactionOutputs, // Applies transaction outputs to stay up-to-date
    ExecuteTransactions,     // Executes transactions to stay up-to-date
}

impl ContinuousSyncingMode {
    pub fn to_label(&self) -> &'static str {
        match self {
            ContinuousSyncingMode::ApplyTransactionOutputs => "apply_transaction_outputs",
            ContinuousSyncingMode::ExecuteTransactions => "execute_transactions",
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StateSyncDriverConfig {
    pub bootstrapping_mode: BootstrappingMode, // The mode by which to bootstrap
    pub commit_notification_timeout_ms: u64, // The max time taken to process a commit notification
    pub continuous_syncing_mode: ContinuousSyncingMode, // The mode by which to sync after bootstrapping
    pub progress_check_interval_ms: u64, // The interval (ms) at which to check state sync progress
    pub max_connection_deadline_secs: u64, // The max time (secs) to wait for connections from peers
    pub max_consecutive_stream_notifications: u64, // The max number of notifications to process per driver loop
    pub max_pending_data_chunks: u64, // The max number of data chunks pending execution or commit
    pub max_stream_wait_time_ms: u64, // The max time (ms) to wait for a data stream notification
    pub num_versions_to_skip_snapshot_sync: u64, // The version lag we'll tolerate before snapshot syncing
}

/// The default state sync driver config will be the one that gets (and keeps)
/// the node up-to-date as quickly and cheaply as possible.
impl Default for StateSyncDriverConfig {
    fn default() -> Self {
        Self {
            bootstrapping_mode: BootstrappingMode::ExecuteTransactionsFromGenesis,
            commit_notification_timeout_ms: 5000,
            continuous_syncing_mode: ContinuousSyncingMode::ExecuteTransactions,
            progress_check_interval_ms: 100,
            max_connection_deadline_secs: 10,
            max_consecutive_stream_notifications: 10,
            max_pending_data_chunks: 100,
            max_stream_wait_time_ms: 5000,
            num_versions_to_skip_snapshot_sync: 100_000_000, // At 5k TPS, this allows a node to fail for about 6 hours.
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StorageServiceConfig {
    pub max_concurrent_requests: u64, // Max num of concurrent storage server tasks
    pub max_epoch_chunk_size: u64,    // Max num of epoch ending ledger infos per chunk
    pub max_lru_cache_size: u64,      // Max num of items in the lru cache before eviction
    pub max_network_channel_size: u64, // Max num of pending network messages
    pub max_network_chunk_bytes: u64, // Max num of bytes to send per network message
    pub max_state_chunk_size: u64,    // Max num of state keys and values per chunk
    pub max_subscription_period_ms: u64, // Max period (ms) of pending subscription requests
    pub max_transaction_chunk_size: u64, // Max num of transactions per chunk
    pub max_transaction_output_chunk_size: u64, // Max num of transaction outputs per chunk
    pub storage_summary_refresh_interval_ms: u64, // The interval (ms) to refresh the storage summary
}

impl Default for StorageServiceConfig {
    fn default() -> Self {
        Self {
            max_concurrent_requests: 4000,
            max_epoch_chunk_size: 100,
            max_lru_cache_size: 500, // At ~0.6MiB per chunk, this should take no more than 0.5GiB
            max_network_channel_size: 4000,
            max_network_chunk_bytes: MAX_MESSAGE_SIZE as u64,
            max_state_chunk_size: 2000,
            max_subscription_period_ms: 5000,
            max_transaction_chunk_size: 2000,
            max_transaction_output_chunk_size: 2000,
            storage_summary_refresh_interval_ms: 50,
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct DataStreamingServiceConfig {
    // The interval (milliseconds) at which to refresh the global data summary.
    pub global_summary_refresh_interval_ms: u64,

    // Maximum number of concurrent data client requests (per stream).
    pub max_concurrent_requests: u64,

    // Maximum number of concurrent data client requests (per stream) for state keys/values.
    pub max_concurrent_state_requests: u64,

    // Maximum channel sizes for each data stream listener. If messages are not
    // consumed, they will be dropped (oldest messages first). The remaining
    // messages will be retrieved using FIFO ordering.
    pub max_data_stream_channel_sizes: u64,

    // Maximum number of retries for a single client request before a data
    // stream will terminate.
    pub max_request_retry: u64,

    // Maximum number of notification ID to response context mappings held in
    // memory. Once the number grows beyond this value, garbage collection occurs.
    pub max_notification_id_mappings: u64,

    // The interval (milliseconds) at which to check the progress of each stream.
    pub progress_check_interval_ms: u64,
}

impl Default for DataStreamingServiceConfig {
    fn default() -> Self {
        Self {
            global_summary_refresh_interval_ms: 50,
            max_concurrent_requests: 3,
            max_concurrent_state_requests: 6,
            max_data_stream_channel_sizes: 300,
            max_request_retry: 3,
            max_notification_id_mappings: 300,
            progress_check_interval_ms: 100,
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct AptosDataClientConfig {
    pub max_num_in_flight_priority_polls: u64, // Max num of in-flight polls for priority peers
    pub max_num_in_flight_regular_polls: u64,  // Max num of in-flight polls for regular peers
    pub response_timeout_ms: u64, // Timeout (in milliseconds) when waiting for a response
    pub summary_poll_interval_ms: u64, // Interval (in milliseconds) between data summary polls
    pub use_compression: bool,    // Whether or not to request compression for incoming data
}

impl Default for AptosDataClientConfig {
    fn default() -> Self {
        Self {
            max_num_in_flight_priority_polls: 10,
            max_num_in_flight_regular_polls: 10,
            response_timeout_ms: 10000,
            summary_poll_interval_ms: 200,
            use_compression: true,
        }
    }
}
<commit_msg>[State Sync] Small tweaks to configs.<commit_after>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use serde::{Deserialize, Serialize};

// The maximum message size per state sync message
pub const MAX_MESSAGE_SIZE: usize = 16 * 1024 * 1024; /* 16 MiB */

#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StateSyncConfig {
    pub data_streaming_service: DataStreamingServiceConfig,
    pub aptos_data_client: AptosDataClientConfig,
    pub state_sync_driver: StateSyncDriverConfig,
    pub storage_service: StorageServiceConfig,
}

/// The bootstrapping mode determines how the node will bootstrap to the latest
/// blockchain state, e.g., directly download the latest states.
#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
pub enum BootstrappingMode {
    ApplyTransactionOutputsFromGenesis, // Applies transaction outputs (starting at genesis)
    DownloadLatestStates, // Downloads the state keys and values (at the latest version)
    ExecuteTransactionsFromGenesis, // Executes transactions (starting at genesis)
}

impl BootstrappingMode {
    pub fn to_label(&self) -> &'static str {
        match self {
            BootstrappingMode::ApplyTransactionOutputsFromGenesis => {
                "apply_transaction_outputs_from_genesis"
            }
            BootstrappingMode::DownloadLatestStates => "download_latest_states",
            BootstrappingMode::ExecuteTransactionsFromGenesis => {
                "execute_transactions_from_genesis"
            }
        }
    }
}

/// The continuous syncing mode determines how the node will stay up-to-date
/// once it has bootstrapped and the blockchain continues to grow, e.g.,
/// continuously executing all transactions.
#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
pub enum ContinuousSyncingMode {
    ApplyTransactionOutputs, // Applies transaction outputs to stay up-to-date
    ExecuteTransactions,     // Executes transactions to stay up-to-date
}

impl ContinuousSyncingMode {
    pub fn to_label(&self) -> &'static str {
        match self {
            ContinuousSyncingMode::ApplyTransactionOutputs => "apply_transaction_outputs",
            ContinuousSyncingMode::ExecuteTransactions => "execute_transactions",
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StateSyncDriverConfig {
    pub bootstrapping_mode: BootstrappingMode, // The mode by which to bootstrap
    pub commit_notification_timeout_ms: u64, // The max time taken to process a commit notification
    pub continuous_syncing_mode: ContinuousSyncingMode, // The mode by which to sync after bootstrapping
    pub progress_check_interval_ms: u64, // The interval (ms) at which to check state sync progress
    pub max_connection_deadline_secs: u64, // The max time (secs) to wait for connections from peers
    pub max_consecutive_stream_notifications: u64, // The max number of notifications to process per driver loop
    pub max_pending_data_chunks: u64, // The max number of data chunks pending execution or commit
    pub max_stream_wait_time_ms: u64, // The max time (ms) to wait for a data stream notification
    pub num_versions_to_skip_snapshot_sync: u64, // The version lag we'll tolerate before snapshot syncing
}

/// The default state sync driver config will be the one that gets (and keeps)
/// the node up-to-date as quickly and cheaply as possible.
impl Default for StateSyncDriverConfig {
    fn default() -> Self {
        Self {
            bootstrapping_mode: BootstrappingMode::ApplyTransactionOutputsFromGenesis,
            commit_notification_timeout_ms: 5000,
            continuous_syncing_mode: ContinuousSyncingMode::ApplyTransactionOutputs,
            progress_check_interval_ms: 100,
            max_connection_deadline_secs: 10,
            max_consecutive_stream_notifications: 10,
            max_pending_data_chunks: 100,
            max_stream_wait_time_ms: 5000,
            num_versions_to_skip_snapshot_sync: 100_000_000, // At 5k TPS, this allows a node to fail for about 6 hours.
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StorageServiceConfig {
    pub max_concurrent_requests: u64, // Max num of concurrent storage server tasks
    pub max_epoch_chunk_size: u64,    // Max num of epoch ending ledger infos per chunk
    pub max_lru_cache_size: u64,      // Max num of items in the lru cache before eviction
    pub max_network_channel_size: u64, // Max num of pending network messages
    pub max_network_chunk_bytes: u64, // Max num of bytes to send per network message
    pub max_state_chunk_size: u64,    // Max num of state keys and values per chunk
    pub max_subscription_period_ms: u64, // Max period (ms) of pending subscription requests
    pub max_transaction_chunk_size: u64, // Max num of transactions per chunk
    pub max_transaction_output_chunk_size: u64, // Max num of transaction outputs per chunk
    pub storage_summary_refresh_interval_ms: u64, // The interval (ms) to refresh the storage summary
}

impl Default for StorageServiceConfig {
    fn default() -> Self {
        Self {
            max_concurrent_requests: 4000,
            max_epoch_chunk_size: 100,
            max_lru_cache_size: 500, // At ~0.6MiB per chunk, this should take no more than 0.5GiB
            max_network_channel_size: 4000,
            max_network_chunk_bytes: MAX_MESSAGE_SIZE as u64,
            max_state_chunk_size: 2000,
            max_subscription_period_ms: 5000,
            max_transaction_chunk_size: 2000,
            max_transaction_output_chunk_size: 2000,
            storage_summary_refresh_interval_ms: 50,
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct DataStreamingServiceConfig {
    // The interval (milliseconds) at which to refresh the global data summary.
    pub global_summary_refresh_interval_ms: u64,

    // Maximum number of concurrent data client requests (per stream).
    pub max_concurrent_requests: u64,

    // Maximum number of concurrent data client requests (per stream) for state keys/values.
    pub max_concurrent_state_requests: u64,

    // Maximum channel sizes for each data stream listener. If messages are not
    // consumed, they will be dropped (oldest messages first). The remaining
    // messages will be retrieved using FIFO ordering.
    pub max_data_stream_channel_sizes: u64,

    // Maximum number of retries for a single client request before a data
    // stream will terminate.
    pub max_request_retry: u64,

    // Maximum number of notification ID to response context mappings held in
    // memory. Once the number grows beyond this value, garbage collection occurs.
    pub max_notification_id_mappings: u64,

    // The interval (milliseconds) at which to check the progress of each stream.
    pub progress_check_interval_ms: u64,
}

impl Default for DataStreamingServiceConfig {
    fn default() -> Self {
        Self {
            global_summary_refresh_interval_ms: 50,
            max_concurrent_requests: 3,
            max_concurrent_state_requests: 6,
            max_data_stream_channel_sizes: 300,
            max_request_retry: 5,
            max_notification_id_mappings: 300,
            progress_check_interval_ms: 100,
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct AptosDataClientConfig {
    pub max_num_in_flight_priority_polls: u64, // Max num of in-flight polls for priority peers
    pub max_num_in_flight_regular_polls: u64,  // Max num of in-flight polls for regular peers
    pub response_timeout_ms: u64, // Timeout (in milliseconds) when waiting for a response
    pub summary_poll_interval_ms: u64, // Interval (in milliseconds) between data summary polls
    pub use_compression: bool,    // Whether or not to request compression for incoming data
}

impl Default for AptosDataClientConfig {
    fn default() -> Self {
        Self {
            max_num_in_flight_priority_polls: 10,
            max_num_in_flight_regular_polls: 10,
            response_timeout_ms: 10000,
            summary_poll_interval_ms: 200,
            use_compression: true,
        }
    }
}
<|endoftext|>


<commit_before>// Copyright © 2017, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

extern crate bragi;
extern crate iron_test;
extern crate serde_json;
use super::BragiHandler;
use super::{count_types, get_types, to_json, get_value};
use super::get_values;
use hyper::status::StatusCode::{BadRequest, NotFound};

pub fn bragi_filter_types_test(es_wrapper: ::ElasticSearchWrapper) {
    let bragi = BragiHandler::new(format!("{}/munin", es_wrapper.host()));

    // ******************************************
    // we the OSM dataset, three-cities bano dataset and a stop file
    // the current dataset are thus (load order matters):
    // - osm_fixture.osm.pbf
    // - bano-three_cities
    // - stops.txt
    // ******************************************
    let osm2mimir = concat!(env!("OUT_DIR"), "/../../../osm2mimir");
    ::launch_and_assert(osm2mimir,
                        vec!["--input=./tests/fixtures/osm_fixture.osm.pbf".into(),
                             "--import-way".into(),
                             "--import-admin".into(),
                             "--import-poi".into(),
                             "--level=8".into(),
                             format!("--connection-string={}", es_wrapper.host())],
                        &es_wrapper);

    let bano2mimir = concat!(env!("OUT_DIR"), "/../../../bano2mimir");
    ::launch_and_assert(bano2mimir,
                        vec!["--input=./tests/fixtures/bano-three_cities.csv".into(),
                             format!("--connection-string={}", es_wrapper.host())],
                        &es_wrapper);

    let stops2mimir = concat!(env!("OUT_DIR"), "/../../../stops2mimir");
    ::launch_and_assert(stops2mimir,
                        vec!["--input=./tests/fixtures/stops.txt".into(),
                             "--dataset=dataset1".into(),
                             format!("--connection-string={}", es_wrapper.host())],
                        &es_wrapper);

    no_type_no_dataset_test(&bragi);
    type_stop_area_no_dataset_test(&bragi);
    type_poi_and_dataset_test(&bragi);
    type_poi_and_city_no_dataset_test(&bragi);
    type_poi_and_city_with_percent_encoding_no_dataset_test(&bragi);
    type_stop_area_dataset_test(&bragi);
    unvalid_type_test(&bragi);
    addr_by_id_test(&bragi);
    admin_by_id_test(&bragi);
    street_by_id_test(&bragi);
    stop_by_id_test(&bragi);
    stop_by_id_not_exist_test(&bragi);
}

fn no_type_no_dataset_test(bragi: &BragiHandler) {
    // with this query we should not find any stops
    let response = bragi.get("/autocomplete?q=Parking vélo Saint-Martin");
    let types = get_types(&response);
    let count = count_types(&types, "public_transport:stop_area");
    assert_eq!(count, 0);
}

fn type_stop_area_no_dataset_test(bragi: &BragiHandler) {
    // with this query we should return an empty response
    let response =
        bragi.get("/autocomplete?q=Parking vélo Saint-Martin&type[]=public_transport:stop_area");
    assert!(response.is_empty());
}

fn type_poi_and_dataset_test(bragi: &BragiHandler) {
    // with this query we should only find pois
    let response =
        bragi.get("/autocomplete?q=Parking vélo Saint-Martin&pt_dataset=dataset1&type[]=poi");
    let types = get_types(&response);
    assert_eq!(count_types(&types, "public_transport:stop_area"), 0);
    assert_eq!(count_types(&types, "city"), 0);
    assert_eq!(count_types(&types, "street"), 0);
    assert_eq!(count_types(&types, "house"), 0);
    assert!(count_types(&types, "poi") > 0);

    let poi = response.first().unwrap();
    assert_eq!(get_value(poi, "name"), "Parking vélo");
}

fn type_poi_and_city_no_dataset_test(bragi: &BragiHandler) {
    // with this query we should only find pois and cities
    let response = bragi.get("/autocomplete?q=melun&type[]=poi&type[]=city");
    let types = get_types(&response);
    assert_eq!(count_types(&types, "public_transport:stop_area"), 0);
    assert_eq!(count_types(&types, "street"), 0);
    assert_eq!(count_types(&types, "house"), 0);
    assert!(count_types(&types, "city") > 0);
    assert!(count_types(&types, "poi") > 0);
}

fn type_poi_and_city_with_percent_encoding_no_dataset_test(bragi: &BragiHandler) {
    // Same test as before but with percent encoded type param
    let response = bragi.get("/autocomplete?q=melun&type%5B%5D=poi&type%5B%5D=city");
    let types = get_types(&response);
    assert_eq!(count_types(&types, "public_transport:stop_area"), 0);
    assert_eq!(count_types(&types, "street"), 0);
    assert_eq!(count_types(&types, "house"), 0);
    assert!(count_types(&types, "city") > 0);
    assert!(count_types(&types, "poi") > 0);
}

fn type_stop_area_dataset_test(bragi: &BragiHandler) {
    // with this query we should only find stop areas
    let response =
        bragi.get("/autocomplete?q=Vaux-le-Pénil&pt_dataset=dataset1&type[]=public_transport:\
                   stop_area");
    let types = get_types(&response);
    assert!(count_types(&types, "public_transport:stop_area") > 0);
    assert_eq!(count_types(&types, "street"), 0);
    assert_eq!(count_types(&types, "house"), 0);
    assert_eq!(count_types(&types, "city"), 0);
    assert_eq!(count_types(&types, "poi"), 0);
}

fn unvalid_type_test(bragi: &BragiHandler) {
    let response = bragi.raw_get("/autocomplete?q=melun&type[]=unvalid");
    assert!(response.is_err());

    let iron_error = response.unwrap_err();
    assert_eq!(iron_error.response.status.unwrap(), BadRequest);

    let json = to_json(iron_error.response);
    let error_msg = json.pointer("/long")
        .unwrap()
        .as_str()
        .unwrap();

    assert!(error_msg.contains("unvalid is not a valid type"))
}

fn admin_by_id_test(bragi: &BragiHandler) {
    let all_20 = bragi.get("/features/admin:fr:77288");
    assert_eq!(all_20.len(), 1);
    let types = get_types(&all_20);
    let count = count_types(&types, "city");
    assert_eq!(count, 1);

    assert_eq!(get_values(&all_20, "id"), vec!["admin:fr:77288"]);
}

fn street_by_id_test(bragi: &BragiHandler) {
    let all_20 = bragi.get("/features/161162362");
    assert_eq!(all_20.len(), 1);
    let types = get_types(&all_20);

    let count = count_types(&types, "street");
    assert_eq!(count, 1);

    assert_eq!(get_values(&all_20, "id"), vec!["161162362"]);
}

fn addr_by_id_test(bragi: &BragiHandler) {
    let all_20 = bragi.get("/features/addr:2.68385;48.50539");
    assert_eq!(all_20.len(), 1);
    let types = get_types(&all_20);
    let count = count_types(&types, "house");
    assert_eq!(count, 1);
    assert_eq!(get_values(&all_20, "id"), vec!["addr:2.68385;48.50539"]);
}

fn stop_by_id_test(bragi: &BragiHandler) {
    // search with id
    let response = bragi.get("/features/stop_area:SA:second_station?pt_dataset=dataset1");
    assert_eq!(response.len(), 1);
    let stop = response.first().unwrap();
    assert_eq!(get_value(stop, "id"), "stop_area:SA:second_station");

}

fn stop_by_id_not_exist_test(bragi: &BragiHandler) {
    // search with id
    let response = bragi.raw_get("/features/stop_area:SA:second_station::AA?pt_dataset=dataset1")
        .unwrap();

    assert_eq!(response.status, Some(NotFound));

    let result_body = iron_test::response::extract_body_to_string(response);
    assert!(result_body.contains("Unable to find object"));
}
<commit_msg>renamme function<commit_after>// Copyright © 2017, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

extern crate bragi;
extern crate iron_test;
extern crate serde_json;
use super::BragiHandler;
use super::{count_types, get_types, to_json, get_value};
use super::get_values;
use hyper::status::StatusCode::{BadRequest, NotFound};

pub fn bragi_filter_types_test(es_wrapper: ::ElasticSearchWrapper) {
    let bragi = BragiHandler::new(format!("{}/munin", es_wrapper.host()));

    // ******************************************
    // we the OSM dataset, three-cities bano dataset and a stop file
    // the current dataset are thus (load order matters):
    // - osm_fixture.osm.pbf
    // - bano-three_cities
    // - stops.txt
    // ******************************************
    let osm2mimir = concat!(env!("OUT_DIR"), "/../../../osm2mimir");
    ::launch_and_assert(osm2mimir,
                        vec!["--input=./tests/fixtures/osm_fixture.osm.pbf".into(),
                             "--import-way".into(),
                             "--import-admin".into(),
                             "--import-poi".into(),
                             "--level=8".into(),
                             format!("--connection-string={}", es_wrapper.host())],
                        &es_wrapper);

    let bano2mimir = concat!(env!("OUT_DIR"), "/../../../bano2mimir");
    ::launch_and_assert(bano2mimir,
                        vec!["--input=./tests/fixtures/bano-three_cities.csv".into(),
                             format!("--connection-string={}", es_wrapper.host())],
                        &es_wrapper);

    let stops2mimir = concat!(env!("OUT_DIR"), "/../../../stops2mimir");
    ::launch_and_assert(stops2mimir,
                        vec!["--input=./tests/fixtures/stops.txt".into(),
                             "--dataset=dataset1".into(),
                             format!("--connection-string={}", es_wrapper.host())],
                        &es_wrapper);

    no_type_no_dataset_test(&bragi);
    type_stop_area_no_dataset_test(&bragi);
    type_poi_and_dataset_test(&bragi);
    type_poi_and_city_no_dataset_test(&bragi);
    type_poi_and_city_with_percent_encoding_no_dataset_test(&bragi);
    type_stop_area_dataset_test(&bragi);
    unvalid_type_test(&bragi);
    addr_by_id_test(&bragi);
    admin_by_id_test(&bragi);
    street_by_id_test(&bragi);
    stop_by_id_test(&bragi);
    stop_area_that_does_not_exists(&bragi);
}

fn no_type_no_dataset_test(bragi: &BragiHandler) {
    // with this query we should not find any stops
    let response = bragi.get("/autocomplete?q=Parking vélo Saint-Martin");
    let types = get_types(&response);
    let count = count_types(&types, "public_transport:stop_area");
    assert_eq!(count, 0);
}

fn type_stop_area_no_dataset_test(bragi: &BragiHandler) {
    // with this query we should return an empty response
    let response =
        bragi.get("/autocomplete?q=Parking vélo Saint-Martin&type[]=public_transport:stop_area");
    assert!(response.is_empty());
}

fn type_poi_and_dataset_test(bragi: &BragiHandler) {
    // with this query we should only find pois
    let response =
        bragi.get("/autocomplete?q=Parking vélo Saint-Martin&pt_dataset=dataset1&type[]=poi");
    let types = get_types(&response);
    assert_eq!(count_types(&types, "public_transport:stop_area"), 0);
    assert_eq!(count_types(&types, "city"), 0);
    assert_eq!(count_types(&types, "street"), 0);
    assert_eq!(count_types(&types, "house"), 0);
    assert!(count_types(&types, "poi") > 0);

    let poi = response.first().unwrap();
    assert_eq!(get_value(poi, "name"), "Parking vélo");
}

fn type_poi_and_city_no_dataset_test(bragi: &BragiHandler) {
    // with this query we should only find pois and cities
    let response = bragi.get("/autocomplete?q=melun&type[]=poi&type[]=city");
    let types = get_types(&response);
    assert_eq!(count_types(&types, "public_transport:stop_area"), 0);
    assert_eq!(count_types(&types, "street"), 0);
    assert_eq!(count_types(&types, "house"), 0);
    assert!(count_types(&types, "city") > 0);
    assert!(count_types(&types, "poi") > 0);
}

fn type_poi_and_city_with_percent_encoding_no_dataset_test(bragi: &BragiHandler) {
    // Same test as before but with percent encoded type param
    let response = bragi.get("/autocomplete?q=melun&type%5B%5D=poi&type%5B%5D=city");
    let types = get_types(&response);
    assert_eq!(count_types(&types, "public_transport:stop_area"), 0);
    assert_eq!(count_types(&types, "street"), 0);
    assert_eq!(count_types(&types, "house"), 0);
    assert!(count_types(&types, "city") > 0);
    assert!(count_types(&types, "poi") > 0);
}

fn type_stop_area_dataset_test(bragi: &BragiHandler) {
    // with this query we should only find stop areas
    let response =
        bragi.get("/autocomplete?q=Vaux-le-Pénil&pt_dataset=dataset1&type[]=public_transport:\
                   stop_area");
    let types = get_types(&response);
    assert!(count_types(&types, "public_transport:stop_area") > 0);
    assert_eq!(count_types(&types, "street"), 0);
    assert_eq!(count_types(&types, "house"), 0);
    assert_eq!(count_types(&types, "city"), 0);
    assert_eq!(count_types(&types, "poi"), 0);
}

fn unvalid_type_test(bragi: &BragiHandler) {
    let response = bragi.raw_get("/autocomplete?q=melun&type[]=unvalid");
    assert!(response.is_err());

    let iron_error = response.unwrap_err();
    assert_eq!(iron_error.response.status.unwrap(), BadRequest);

    let json = to_json(iron_error.response);
    let error_msg = json.pointer("/long")
        .unwrap()
        .as_str()
        .unwrap();

    assert!(error_msg.contains("unvalid is not a valid type"))
}

fn admin_by_id_test(bragi: &BragiHandler) {
    let all_20 = bragi.get("/features/admin:fr:77288");
    assert_eq!(all_20.len(), 1);
    let types = get_types(&all_20);
    let count = count_types(&types, "city");
    assert_eq!(count, 1);

    assert_eq!(get_values(&all_20, "id"), vec!["admin:fr:77288"]);
}

fn street_by_id_test(bragi: &BragiHandler) {
    let all_20 = bragi.get("/features/161162362");
    assert_eq!(all_20.len(), 1);
    let types = get_types(&all_20);

    let count = count_types(&types, "street");
    assert_eq!(count, 1);

    assert_eq!(get_values(&all_20, "id"), vec!["161162362"]);
}

fn addr_by_id_test(bragi: &BragiHandler) {
    let all_20 = bragi.get("/features/addr:2.68385;48.50539");
    assert_eq!(all_20.len(), 1);
    let types = get_types(&all_20);
    let count = count_types(&types, "house");
    assert_eq!(count, 1);
    assert_eq!(get_values(&all_20, "id"), vec!["addr:2.68385;48.50539"]);
}

fn stop_by_id_test(bragi: &BragiHandler) {
    // search with id
    let response = bragi.get("/features/stop_area:SA:second_station?pt_dataset=dataset1");
    assert_eq!(response.len(), 1);
    let stop = response.first().unwrap();
    assert_eq!(get_value(stop, "id"), "stop_area:SA:second_station");

}

fn stop_area_that_does_not_exists(bragi: &BragiHandler) {
    // search with id
    let response = bragi.raw_get("/features/stop_area:SA:second_station::AA?pt_dataset=dataset1")
        .unwrap();

    assert_eq!(response.status, Some(NotFound));

    let result_body = iron_test::response::extract_body_to_string(response);
    assert!(result_body.contains("Unable to find object"));
}
<|endoftext|>


<commit_before>//! Licensed under the Apache License, Version 2.0
//! http://www.apache.org/licenses/LICENSE-2.0 or the MIT license
//! http://opensource.org/licenses/MIT, at your
//! option. This file may not be copied, modified, or distributed
//! except according to those terms.

use std::num::Saturating;

/// Alternate elements from two iterators until both
/// are run out
#[deriving(Clone)]
pub struct Interleave<I, J> {
    a: I,
    b: J,
    flag: bool,
}

impl<I, J> Interleave<I, J> {
    ///
    pub fn new(a: I, b: J) -> Interleave<I, J> {
        Interleave{a: a, b: b, flag: false}
    }
}

impl<A, I: Iterator<A>, J: Iterator<A>> Iterator<A> for Interleave<I, J> {
    #[inline]
    fn next(&mut self) -> Option<A> {
        self.flag = !self.flag;
        if self.flag {
            match self.a.next() {
                None => self.b.next(),
                r => r,
            }
        } else {
            match self.b.next() {
                None => self.a.next(),
                r => r,
            }
        }
    }
}

/// Clonable iterator adaptor to map elementwise
/// from `Iterator<A>` to `Iterator<B>`
///
/// Created with `.static_map(..)` on an iterator
pub struct FnMap<A, B, I> {
    map: fn(A) -> B,
    iter: I,
}

impl<A, B, I> FnMap<A, B, I>
{
    pub fn new(iter: I, map: fn(A) -> B) -> FnMap<A, B, I> {
        FnMap{iter: iter, map: map}
    }
}

impl<A, B, I: Iterator<A>> Iterator<B> for FnMap<A, B, I>
{
    #[inline]
    fn next(&mut self) -> Option<B> {
        self.iter.next().map(|a| (self.map)(a))
    }

    fn size_hint(&self) -> (uint, Option<uint>) {
        self.iter.size_hint()
    }
}

impl<A, B, I: DoubleEndedIterator<A>> DoubleEndedIterator<B>
for FnMap<A, B, I>
{
    #[inline]
    fn next_back(&mut self) -> Option<B> {
        self.iter.next_back().map(|a| (self.map)(a))
    }
}

impl<A, B, I: Clone> Clone for FnMap<A, B, I>
{
    fn clone(&self) -> FnMap<A, B, I> {
        FnMap::new(self.iter.clone(), self.map)
    }
}

/// An iterator adaptor that allows putting back a single
/// item to the front of the iterator.
#[deriving(Clone)]
pub struct PutBack<A, T> {
    top: Option<A>,
    iter: T
}

impl<A, T> PutBack<A, T> {
    ///
    #[inline]
    pub fn new(it: T) -> PutBack<A, T> {
        PutBack{top: None, iter: it}
    }

    /// Put back a single value to the front of the iterator.
    ///
    /// If a value is already in the put back slot, it is overwritten.
    #[inline]
    pub fn put_back(&mut self, x: A) {
        self.top = Some(x)
    }
}

impl<A, T: Iterator<A>> Iterator<A> for PutBack<A, T> {
    #[inline]
    fn next(&mut self) -> Option<A> {
        match self.top.take() {
            None => self.iter.next(),
            top => top,
        }
    }
    #[inline]
    fn size_hint(&self) -> (uint, Option<uint>) {
        let (lo, hi) = self.iter.size_hint();
        match self.top {
            Some(_) => (lo.saturating_add(1), hi.and_then(|x| x.checked_add(&1))),
            None => (lo, hi)
        }
    }
}

/// An iterator adaptor that iterates over the cartesian product of
/// the element sets of two iterators `I` and `J`.
#[deriving(Clone)]
pub struct Product<A, I, J> {
    a: I,
    a_cur: Option<A>,
    b: J,
    b_orig: J,
}

impl<A: Clone, B, I: Iterator<A>, J: Clone + Iterator<B>>
    Product<A, I, J>
{
    pub fn new(i: I, j: J) -> Product<A, I, J>
    {
        let mut i = i;
        Product{a_cur: i.next(), a: i, b: j.clone(), b_orig: j}
    }
}

impl<A: Clone, I: Iterator<A>, B, J: Clone + Iterator<B>>
Iterator<(A, B)> for Product<A, I, J>
{
    fn next(&mut self) -> Option<(A, B)>
    {
        let elt_b = match self.b.next() {
            None => {
                self.b = self.b_orig.clone();
                match self.b.next() {
                    None => return None,
                    Some(x) => {
                        self.a_cur = self.a.next();
                        x
                    }
                }
            }
            Some(x) => x
        };
        match self.a_cur {
            None => None,
            Some(ref a) => {
                Some((a.clone(), elt_b))
            }
        }
    }

    fn size_hint(&self) -> (uint, Option<uint>)
    {
        let (a, ah) = self.a.size_hint();
        let (b, bh) = self.b.size_hint();
        let (bo, boh) = self.b_orig.size_hint();

        // Compute a * bo + b for both lower and upper bound
        let low = a.checked_mul(&bo)
                    .and_then(|x| x.checked_add(&b))
                    .unwrap_or(::std::uint::MAX);
        let high = ah.and_then(|x| boh.and_then(|y| x.checked_mul(&y)))
                     .and_then(|x| bh.and_then(|y| x.checked_add(&y)));
        (low, high)
    }
}

<commit_msg>Update doc for FnMap<commit_after>//! Licensed under the Apache License, Version 2.0
//! http://www.apache.org/licenses/LICENSE-2.0 or the MIT license
//! http://opensource.org/licenses/MIT, at your
//! option. This file may not be copied, modified, or distributed
//! except according to those terms.

use std::num::Saturating;

/// Alternate elements from two iterators until both
/// are run out
#[deriving(Clone)]
pub struct Interleave<I, J> {
    a: I,
    b: J,
    flag: bool,
}

impl<I, J> Interleave<I, J> {
    ///
    pub fn new(a: I, b: J) -> Interleave<I, J> {
        Interleave{a: a, b: b, flag: false}
    }
}

impl<A, I: Iterator<A>, J: Iterator<A>> Iterator<A> for Interleave<I, J> {
    #[inline]
    fn next(&mut self) -> Option<A> {
        self.flag = !self.flag;
        if self.flag {
            match self.a.next() {
                None => self.b.next(),
                r => r,
            }
        } else {
            match self.b.next() {
                None => self.a.next(),
                r => r,
            }
        }
    }
}

/// Clonable iterator adaptor to map elementwise
/// from `Iterator<A>` to `Iterator<B>`
///
/// Created with the `.fn_map(..)` method on an iterator
pub struct FnMap<A, B, I> {
    map: fn(A) -> B,
    iter: I,
}

impl<A, B, I> FnMap<A, B, I>
{
    pub fn new(iter: I, map: fn(A) -> B) -> FnMap<A, B, I> {
        FnMap{iter: iter, map: map}
    }
}

impl<A, B, I: Iterator<A>> Iterator<B> for FnMap<A, B, I>
{
    #[inline]
    fn next(&mut self) -> Option<B> {
        self.iter.next().map(|a| (self.map)(a))
    }

    fn size_hint(&self) -> (uint, Option<uint>) {
        self.iter.size_hint()
    }
}

impl<A, B, I: DoubleEndedIterator<A>> DoubleEndedIterator<B>
for FnMap<A, B, I>
{
    #[inline]
    fn next_back(&mut self) -> Option<B> {
        self.iter.next_back().map(|a| (self.map)(a))
    }
}

impl<A, B, I: Clone> Clone for FnMap<A, B, I>
{
    fn clone(&self) -> FnMap<A, B, I> {
        FnMap::new(self.iter.clone(), self.map)
    }
}

/// An iterator adaptor that allows putting back a single
/// item to the front of the iterator.
#[deriving(Clone)]
pub struct PutBack<A, T> {
    top: Option<A>,
    iter: T
}

impl<A, T> PutBack<A, T> {
    ///
    #[inline]
    pub fn new(it: T) -> PutBack<A, T> {
        PutBack{top: None, iter: it}
    }

    /// Put back a single value to the front of the iterator.
    ///
    /// If a value is already in the put back slot, it is overwritten.
    #[inline]
    pub fn put_back(&mut self, x: A) {
        self.top = Some(x)
    }
}

impl<A, T: Iterator<A>> Iterator<A> for PutBack<A, T> {
    #[inline]
    fn next(&mut self) -> Option<A> {
        match self.top.take() {
            None => self.iter.next(),
            top => top,
        }
    }
    #[inline]
    fn size_hint(&self) -> (uint, Option<uint>) {
        let (lo, hi) = self.iter.size_hint();
        match self.top {
            Some(_) => (lo.saturating_add(1), hi.and_then(|x| x.checked_add(&1))),
            None => (lo, hi)
        }
    }
}

/// An iterator adaptor that iterates over the cartesian product of
/// the element sets of two iterators `I` and `J`.
#[deriving(Clone)]
pub struct Product<A, I, J> {
    a: I,
    a_cur: Option<A>,
    b: J,
    b_orig: J,
}

impl<A: Clone, B, I: Iterator<A>, J: Clone + Iterator<B>>
    Product<A, I, J>
{
    pub fn new(i: I, j: J) -> Product<A, I, J>
    {
        let mut i = i;
        Product{a_cur: i.next(), a: i, b: j.clone(), b_orig: j}
    }
}

impl<A: Clone, I: Iterator<A>, B, J: Clone + Iterator<B>>
Iterator<(A, B)> for Product<A, I, J>
{
    fn next(&mut self) -> Option<(A, B)>
    {
        let elt_b = match self.b.next() {
            None => {
                self.b = self.b_orig.clone();
                match self.b.next() {
                    None => return None,
                    Some(x) => {
                        self.a_cur = self.a.next();
                        x
                    }
                }
            }
            Some(x) => x
        };
        match self.a_cur {
            None => None,
            Some(ref a) => {
                Some((a.clone(), elt_b))
            }
        }
    }

    fn size_hint(&self) -> (uint, Option<uint>)
    {
        let (a, ah) = self.a.size_hint();
        let (b, bh) = self.b.size_hint();
        let (bo, boh) = self.b_orig.size_hint();

        // Compute a * bo + b for both lower and upper bound
        let low = a.checked_mul(&bo)
                    .and_then(|x| x.checked_add(&b))
                    .unwrap_or(::std::uint::MAX);
        let high = ah.and_then(|x| boh.and_then(|y| x.checked_mul(&y)))
                     .and_then(|x| bh.and_then(|y| x.checked_add(&y)));
        (low, high)
    }
}

<|endoftext|>


<commit_before>//! The Lea compiler command line frontend

#![feature(libc)]

extern crate lea;
extern crate term;
extern crate libc;

use lea::compiler::*;
use lea::compiler::span::*;

use std::env;
use std::fmt;
use std::default::Default;
use std::io::{self, Read, Write, Stderr};
use std::fs::File;
use std::path::Path;

macro_rules! printerr {
    ($($arg:tt)*) => {
        print_err_fmt(format_args!($($arg)*));
    }
}

#[inline]
#[allow(unused_must_use)]
fn print_err_fmt(fmt: fmt::Arguments) {
    writeln!(&mut io::stderr(), "{}", fmt);
}

fn print_usage() {
    println!("Usage: leac [file]");
    println!("");
    println!("Parses and checks the given file. If file is \"-\", reads stdin.");
    println!("When the compiler is done, this will actually compile the source code.");
}

fn compile(code: &str, filename: &str) -> io::Result<()> {
    let mut term = term::stderr();
    let mut stderr = io::stderr();
    let mut dummy = DummyTerm(&mut stderr);
    let mut fmt_target: &mut term::Terminal<Stderr> = if unsafe {
        libc::isatty(libc::STDERR_FILENO) != 0
    } {
        // we can print to the term (or at least try to)
        match term {
            Some(ref mut term) => &mut **term,
            None => &mut dummy,
        }
    } else {
        // use dummy, since we don't want escape sequences
        &mut dummy
    };

    match compile_str(code, filename, &CompileConfig::default()) {
        Err(e) => match e {
            ErrParse(err) => {
                try!(err.format(code.as_ref(), filename, fmt_target));
            },
            ErrCheck(errs) => {
                let mut i = 1;
                for err in &errs {
                    try!(err.format(code.as_ref(), filename, fmt_target));
                    if i < errs.len() - 1 { try!(write!(fmt_target, "\n")); }
                    i += 1;
                }
            },
            ErrLint(warns) => {
                let mut first = true;
                for w in &warns {
                    if first { first = false; } else { try!(write!(fmt_target, "\n")); }
                    try!(w.format(code.as_ref(), filename, fmt_target));
                }
            },
            ErrEmit(errs) => {
                for err in &errs {
                    let mut msg = err.msg.to_string();
                    if let Some(ref d) = err.detail {
                        msg.push_str(format!(" ({})", d).as_ref());
                    }

                    try!(fmt_target.fg(term::color::RED));
                    try!(write!(fmt_target, "{}", msg));
                    try!(fmt_target.reset());
                }
            }
        },
        Ok(output) => {
            let warns = output.warns;
            if warns.len() > 0 {
                for w in warns {
                    try!(w.format(code, filename, fmt_target));
                }
            }
        }
    }

    Ok(())
}

fn compile_file(filename: &str) -> io::Result<()> {
    match File::open(&Path::new(filename.clone())) {
        Ok(mut file) => {
            let mut code = String::new();
            match file.read_to_string(&mut code) {
                Ok(_) => {
                    try!(compile(code.as_ref(), filename.as_ref()));
                },
                Err(err) => {
                    printerr!("error reading file \"{}\": {}", filename, err);
                }
            }
        },
        Err(err) => {
            printerr!("error opening file \"{}\": {}", filename, err);
        }
    };

    Ok(())
}

pub fn main() {
    let mut iter = env::args();
    let mut inputarg: Option<String> = None;
    let mut lastarg: Option<String> = None;

    iter.next();    // Consume first argument (exec name)
    for arg in iter {
        if let Some(..) = inputarg {
            printerr!("Invalid argument: {}", lastarg.unwrap());
            print_usage();
            return;
        }

        match arg.as_ref() {
            "--help" | "-h" => {
                print_usage();
                return;
            },
            _ => {
                // Unknown arg, must be input file
                inputarg = Some(arg.clone());
            },
        }

        lastarg = Some(arg);
    }

    if let None = inputarg {
        printerr!("error: no input file");
        print_usage();
        return;
    }

    let f_string = inputarg.unwrap();
    let f: &str = f_string.as_ref();
    if f == "-" {
        let mut code = String::new();
        io::stdin().read_to_string(&mut code).unwrap();

        println!("");   // print newline to seperate leac output from source input
        compile(code.as_ref(), "<stdin>").unwrap();
    } else {
        compile_file(f).unwrap();
    }
}

#[cfg(test)]
mod tests {
    #[test]
    fn test() {
        // TODO
    }
}
<commit_msg>Add --version command to leac<commit_after>//! The Lea compiler command line frontend

#![feature(libc)]

extern crate lea;
extern crate term;
extern crate libc;

use lea::compiler::*;
use lea::compiler::span::*;

use std::env;
use std::fmt;
use std::default::Default;
use std::io::{self, Read, Write, Stderr};
use std::fs::File;
use std::path::Path;

macro_rules! printerr {
    ($($arg:tt)*) => {
        print_err_fmt(format_args!($($arg)*))
    }
}

fn print_err_fmt(fmt: fmt::Arguments) -> io::Result<()> {
    writeln!(&mut io::stderr(), "{}", fmt)
}

fn print_usage() {
    print!("\
usage: leac [options] <file>

Parses and checks the given file. If file is \"-\", reads stdin. When the compiler is done, this will actually compile the source code.

Options:
 -h, --help     Display this help text
 -v, --version  Display the version of the Lea package
");
}

fn print_version() {
    println!("Lea {}", env!("CARGO_PKG_VERSION"));
}

/// Compiles a piece of code. Prints all errors / warnings to stderr, using color if stderr is a
/// terminal.
fn compile(code: &str, filename: &str) -> io::Result<()> {
    let mut term = term::stderr();
    let mut stderr = io::stderr();
    let mut dummy = DummyTerm(&mut stderr);
    let mut fmt_target: &mut term::Terminal<Stderr> = if unsafe {
        libc::isatty(libc::STDERR_FILENO) != 0
    } {
        // we can print to the term (or at least try to)
        match term {
            Some(ref mut term) => &mut **term,
            None => &mut dummy,
        }
    } else {
        // use dummy, since we don't want escape sequences
        &mut dummy
    };

    match compile_str(code, filename, &CompileConfig::default()) {
        Err(e) => match e {
            ErrParse(err) => {
                try!(err.format(code.as_ref(), filename, fmt_target));
            },
            ErrCheck(errs) => {
                let mut i = 1;
                for err in &errs {
                    try!(err.format(code.as_ref(), filename, fmt_target));
                    if i < errs.len() - 1 { try!(write!(fmt_target, "\n")); }
                    i += 1;
                }
            },
            ErrLint(warns) => {
                let mut first = true;
                for w in &warns {
                    if first { first = false; } else { try!(write!(fmt_target, "\n")); }
                    try!(w.format(code.as_ref(), filename, fmt_target));
                }
            },
            ErrEmit(errs) => {
                for err in &errs {
                    let mut msg = err.msg.to_string();
                    if let Some(ref d) = err.detail {
                        msg.push_str(format!(" ({})", d).as_ref());
                    }

                    try!(fmt_target.fg(term::color::RED));
                    try!(write!(fmt_target, "{}", msg));
                    try!(fmt_target.reset());
                }
            }
        },
        Ok(output) => {
            let warns = output.warns;
            if warns.len() > 0 {
                for w in warns {
                    try!(w.format(code, filename, fmt_target));
                }
            }
        }
    }

    Ok(())
}

fn compile_file(filename: &str) -> io::Result<()> {
    match File::open(&Path::new(filename.clone())) {
        Ok(mut file) => {
            let mut code = String::new();
            match file.read_to_string(&mut code) {
                Ok(_) => {
                    compile(code.as_ref(), filename.as_ref())
                },
                Err(err) => {
                    printerr!("error reading file \"{}\": {}", filename, err)
                }
            }
        },
        Err(err) => {
            printerr!("error opening file \"{}\": {}", filename, err)
        }
    }
}

pub fn main() {
    let mut iter = env::args();
    let mut inputarg: Option<String> = None;
    let mut lastarg: Option<String> = None;

    iter.next();    // Consume first argument (exec name)
    for arg in iter {
        if let Some(..) = inputarg {
            printerr!("invalid argument: {}", lastarg.unwrap()).unwrap();
            print_usage();
            return;
        }

        match arg.as_ref() {
            "-h" | "--help" => {
                print_usage();
                return;
            }
            "-v" | "--version" => {
                print_version();
                return;
            }
            _ => {
                // Unknown arg, must be input file
                inputarg = Some(arg.clone());
            }
        }

        lastarg = Some(arg);
    }

    if let None = inputarg {
        printerr!("error: no input file").unwrap();
        print_usage();
        return;
    }

    let f_string = inputarg.unwrap();
    let f: &str = f_string.as_ref();
    if f == "-" {
        let mut code = String::new();
        io::stdin().read_to_string(&mut code).unwrap();

        compile(code.as_ref(), "<stdin>").unwrap();
    } else {
        compile_file(f).unwrap();
    }
}

#[cfg(test)]
mod tests {
    #[test]
    fn test() {
        // TODO
    }
}
<|endoftext|>