<commit_before>use na::Real;
use pipeline::events::{ContactEvent, ContactEvents, ProximityEvent, ProximityEvents};
use pipeline::narrow_phase::{
    ContactAlgorithm, ContactDispatcher, ContactPairs, NarrowPhase, ProximityAlgorithm,
    ProximityDispatcher, ProximityPairs,
};
use pipeline::world::{CollisionObjectHandle, CollisionObjectSlab, GeometricQueryType};
use query::Proximity;
use std::collections::hash_map::Entry;
use std::collections::HashMap;
use utils::{DeterministicState, SortedPair};
use utils::IdAllocator;

// FIXME: move this to the `narrow_phase` module.
/// Collision detector dispatcher for collision objects.
pub struct DefaultNarrowPhase<N> {
    id_alloc: IdAllocator,
    contact_dispatcher: Box<ContactDispatcher<N>>,
    contact_generators:
    HashMap<SortedPair<CollisionObjectHandle>, ContactAlgorithm<N>, DeterministicState>,

    proximity_dispatcher: Box<ProximityDispatcher<N>>,
    proximity_detectors:
    HashMap<SortedPair<CollisionObjectHandle>, ProximityAlgorithm<N>, DeterministicState>,
}

impl<N: 'static> DefaultNarrowPhase<N> {
    /// Creates a new `DefaultNarrowPhase`.
    pub fn new(
        contact_dispatcher: Box<ContactDispatcher<N>>,
        proximity_dispatcher: Box<ProximityDispatcher<N>>,
    ) -> DefaultNarrowPhase<N> {
        DefaultNarrowPhase {
            id_alloc: IdAllocator::new(),
            contact_dispatcher: contact_dispatcher,
            contact_generators: HashMap::with_hasher(DeterministicState::new()),

            proximity_dispatcher: proximity_dispatcher,
            proximity_detectors: HashMap::with_hasher(DeterministicState::new()),
        }
    }
}

impl<N: Real, T> NarrowPhase<N, T> for DefaultNarrowPhase<N> {
    fn update(
        &mut self,
        objects: &CollisionObjectSlab<N, T>,
        contact_events: &mut ContactEvents,
        proximity_events: &mut ProximityEvents,
        timestamp: usize,
    ) {
        for (key, value) in self.contact_generators.iter_mut() {
            let co1 = &objects[key.0];
            let co2 = &objects[key.1];

            if co1.timestamp == timestamp || co2.timestamp == timestamp {
                let had_contacts = value.num_contacts() != 0;

                if let Some(prediction) = co1
                    .query_type()
                    .contact_queries_to_prediction(co2.query_type())
                    {
                        let _ = value.update(
                            &*self.contact_dispatcher,
                            0,
                            &co1.position(),
                            co1.shape().as_ref(),
                            0,
                            &co2.position(),
                            co2.shape().as_ref(),
                            &prediction,
                            &mut self.id_alloc,
                        );
                    } else {
                    panic!("Unable to compute contact between collision objects with query types different from `GeometricQueryType::Contacts(..)`.")
                }

                if value.num_contacts() == 0 {
                    if had_contacts {
                        contact_events.push(ContactEvent::Stopped(co1.handle(), co2.handle()));
                    }
                } else {
                    if !had_contacts {
                        contact_events.push(ContactEvent::Started(co1.handle(), co2.handle()));
                    }
                }
            }
        }

        for (key, value) in self.proximity_detectors.iter_mut() {
            let co1 = &objects[key.0];
            let co2 = &objects[key.1];

            if co1.timestamp == timestamp || co2.timestamp == timestamp {
                let prev_prox = value.proximity();

                let _ = value.update(
                    &*self.proximity_dispatcher,
                    &co1.position(),
                    co1.shape().as_ref(),
                    &co2.position(),
                    co2.shape().as_ref(),
                    co1.query_type().query_limit() + co2.query_type().query_limit(),
                );

                let new_prox = value.proximity();

                if new_prox != prev_prox {
                    proximity_events.push(ProximityEvent::new(
                        co1.handle(),
                        co2.handle(),
                        prev_prox,
                        new_prox,
                    ));
                }
            }
        }
    }

    fn handle_interaction(
        &mut self,
        contact_events: &mut ContactEvents,
        proximity_events: &mut ProximityEvents,
        objects: &CollisionObjectSlab<N, T>,
        handle1: CollisionObjectHandle,
        handle2: CollisionObjectHandle,
        started: bool,
    ) {
        let key = SortedPair::new(handle1, handle2);
        let co1 = &objects[key.0];
        let co2 = &objects[key.1];

        match (co1.query_type(), co2.query_type()) {
            (GeometricQueryType::Contacts(..), GeometricQueryType::Contacts(..)) => {
                if started {
                    let dispatcher = &self.contact_dispatcher;

                    if let Entry::Vacant(entry) = self.contact_generators.entry(key) {
                        if let Some(detector) = dispatcher
                            .get_contact_algorithm(co1.shape().as_ref(), co2.shape().as_ref())
                            {
                                let _ = entry.insert(detector);
                            }
                    }
                } else {
                    // Proximity stopped.
                    if let Some(mut detector) = self.contact_generators.remove(&key) {
                        // Register a collision lost event if there was a contact.
                        if detector.num_contacts() != 0 {
                            contact_events.push(ContactEvent::Stopped(co1.handle(), co2.handle()));
                            detector.clear(&mut self.id_alloc);
                        }
                    }
                }
            }
            (_, GeometricQueryType::Proximity(_)) | (GeometricQueryType::Proximity(_), _) => {
                if started {
                    let dispatcher = &self.proximity_dispatcher;

                    if let Entry::Vacant(entry) = self.proximity_detectors.entry(key) {
                        if let Some(detector) = dispatcher
                            .get_proximity_algorithm(co1.shape().as_ref(), co2.shape().as_ref())
                            {
                                let _ = entry.insert(detector);
                            }
                    }
                } else {
                    // Proximity stopped.
                    if let Some(detector) = self.proximity_detectors.remove(&key) {
                        // Register a proximity lost signal if they were not disjoint.
                        let prev_prox = detector.proximity();

                        if prev_prox != Proximity::Disjoint {
                            let event = ProximityEvent::new(
                                co1.handle(),
                                co2.handle(),
                                prev_prox,
                                Proximity::Disjoint,
                            );
                            proximity_events.push(event);
                        }
                    }
                }
            }
        }
    }

    fn handle_removal(
        &mut self,
        _: &CollisionObjectSlab<N, T>,
        handle1: CollisionObjectHandle,
        handle2: CollisionObjectHandle,
    ) {
        let key = SortedPair::new(handle1, handle2);
        let _ = self.proximity_detectors.remove(&key);

        if let Some(mut gen) = self.contact_generators.remove(&key) {
            gen.clear(&mut self.id_alloc);
        }
    }

    fn contact_pair(
        &self,
        handle1: CollisionObjectHandle,
        handle2: CollisionObjectHandle,
    ) -> Option<&ContactAlgorithm<N>> {
        let key = SortedPair::new(handle1, handle2);
        self.contact_generators.get(&key)
    }

    fn contact_pairs<'a>(
        &'a self,
        objects: &'a CollisionObjectSlab<N, T>,
    ) -> ContactPairs<'a, N, T> {
        ContactPairs::new(objects, self.contact_generators.iter())
    }

    fn proximity_pairs<'a>(
        &'a self,
        objects: &'a CollisionObjectSlab<N, T>,
    ) -> ProximityPairs<'a, N, T> {
        ProximityPairs::new(objects, self.proximity_detectors.iter())
    }
}
<commit_msg>Clear the removed narrow phase detectors even if they have no contact.<commit_after>use na::Real;
use pipeline::events::{ContactEvent, ContactEvents, ProximityEvent, ProximityEvents};
use pipeline::narrow_phase::{
    ContactAlgorithm, ContactDispatcher, ContactPairs, NarrowPhase, ProximityAlgorithm,
    ProximityDispatcher, ProximityPairs,
};
use pipeline::world::{CollisionObjectHandle, CollisionObjectSlab, GeometricQueryType};
use query::Proximity;
use std::collections::hash_map::Entry;
use std::collections::HashMap;
use utils::{DeterministicState, SortedPair};
use utils::IdAllocator;

// FIXME: move this to the `narrow_phase` module.
/// Collision detector dispatcher for collision objects.
pub struct DefaultNarrowPhase<N> {
    id_alloc: IdAllocator,
    contact_dispatcher: Box<ContactDispatcher<N>>,
    contact_generators:
    HashMap<SortedPair<CollisionObjectHandle>, ContactAlgorithm<N>, DeterministicState>,

    proximity_dispatcher: Box<ProximityDispatcher<N>>,
    proximity_detectors:
    HashMap<SortedPair<CollisionObjectHandle>, ProximityAlgorithm<N>, DeterministicState>,
}

impl<N: 'static> DefaultNarrowPhase<N> {
    /// Creates a new `DefaultNarrowPhase`.
    pub fn new(
        contact_dispatcher: Box<ContactDispatcher<N>>,
        proximity_dispatcher: Box<ProximityDispatcher<N>>,
    ) -> DefaultNarrowPhase<N> {
        DefaultNarrowPhase {
            id_alloc: IdAllocator::new(),
            contact_dispatcher: contact_dispatcher,
            contact_generators: HashMap::with_hasher(DeterministicState::new()),

            proximity_dispatcher: proximity_dispatcher,
            proximity_detectors: HashMap::with_hasher(DeterministicState::new()),
        }
    }
}

impl<N: Real, T> NarrowPhase<N, T> for DefaultNarrowPhase<N> {
    fn update(
        &mut self,
        objects: &CollisionObjectSlab<N, T>,
        contact_events: &mut ContactEvents,
        proximity_events: &mut ProximityEvents,
        timestamp: usize,
    ) {
        for (key, value) in self.contact_generators.iter_mut() {
            let co1 = &objects[key.0];
            let co2 = &objects[key.1];

            if co1.timestamp == timestamp || co2.timestamp == timestamp {
                let had_contacts = value.num_contacts() != 0;

                if let Some(prediction) = co1
                    .query_type()
                    .contact_queries_to_prediction(co2.query_type())
                    {
                        let _ = value.update(
                            &*self.contact_dispatcher,
                            0,
                            &co1.position(),
                            co1.shape().as_ref(),
                            0,
                            &co2.position(),
                            co2.shape().as_ref(),
                            &prediction,
                            &mut self.id_alloc,
                        );
                    } else {
                    panic!("Unable to compute contact between collision objects with query types different from `GeometricQueryType::Contacts(..)`.")
                }

                if value.num_contacts() == 0 {
                    if had_contacts {
                        contact_events.push(ContactEvent::Stopped(co1.handle(), co2.handle()));
                    }
                } else {
                    if !had_contacts {
                        contact_events.push(ContactEvent::Started(co1.handle(), co2.handle()));
                    }
                }
            }
        }

        for (key, value) in self.proximity_detectors.iter_mut() {
            let co1 = &objects[key.0];
            let co2 = &objects[key.1];

            if co1.timestamp == timestamp || co2.timestamp == timestamp {
                let prev_prox = value.proximity();

                let _ = value.update(
                    &*self.proximity_dispatcher,
                    &co1.position(),
                    co1.shape().as_ref(),
                    &co2.position(),
                    co2.shape().as_ref(),
                    co1.query_type().query_limit() + co2.query_type().query_limit(),
                );

                let new_prox = value.proximity();

                if new_prox != prev_prox {
                    proximity_events.push(ProximityEvent::new(
                        co1.handle(),
                        co2.handle(),
                        prev_prox,
                        new_prox,
                    ));
                }
            }
        }
    }

    fn handle_interaction(
        &mut self,
        contact_events: &mut ContactEvents,
        proximity_events: &mut ProximityEvents,
        objects: &CollisionObjectSlab<N, T>,
        handle1: CollisionObjectHandle,
        handle2: CollisionObjectHandle,
        started: bool,
    ) {
        let key = SortedPair::new(handle1, handle2);
        let co1 = &objects[key.0];
        let co2 = &objects[key.1];

        match (co1.query_type(), co2.query_type()) {
            (GeometricQueryType::Contacts(..), GeometricQueryType::Contacts(..)) => {
                if started {
                    let dispatcher = &self.contact_dispatcher;

                    if let Entry::Vacant(entry) = self.contact_generators.entry(key) {
                        if let Some(detector) = dispatcher
                            .get_contact_algorithm(co1.shape().as_ref(), co2.shape().as_ref())
                            {
                                let _ = entry.insert(detector);
                            }
                    }
                } else {
                    // Proximity stopped.
                    if let Some(mut detector) = self.contact_generators.remove(&key) {
                        // Register a collision lost event if there was a contact.
                        if detector.num_contacts() != 0 {
                            contact_events.push(ContactEvent::Stopped(co1.handle(), co2.handle()));
                        }

                        detector.clear(&mut self.id_alloc);
                    }
                }
            }
            (_, GeometricQueryType::Proximity(_)) | (GeometricQueryType::Proximity(_), _) => {
                if started {
                    let dispatcher = &self.proximity_dispatcher;

                    if let Entry::Vacant(entry) = self.proximity_detectors.entry(key) {
                        if let Some(detector) = dispatcher
                            .get_proximity_algorithm(co1.shape().as_ref(), co2.shape().as_ref())
                            {
                                let _ = entry.insert(detector);
                            }
                    }
                } else {
                    // Proximity stopped.
                    if let Some(detector) = self.proximity_detectors.remove(&key) {
                        // Register a proximity lost signal if they were not disjoint.
                        let prev_prox = detector.proximity();

                        if prev_prox != Proximity::Disjoint {
                            let event = ProximityEvent::new(
                                co1.handle(),
                                co2.handle(),
                                prev_prox,
                                Proximity::Disjoint,
                            );
                            proximity_events.push(event);
                        }
                    }
                }
            }
        }
    }

    fn handle_removal(
        &mut self,
        _: &CollisionObjectSlab<N, T>,
        handle1: CollisionObjectHandle,
        handle2: CollisionObjectHandle,
    ) {
        let key = SortedPair::new(handle1, handle2);
        let _ = self.proximity_detectors.remove(&key);

        if let Some(mut gen) = self.contact_generators.remove(&key) {
            gen.clear(&mut self.id_alloc);
        }
    }

    fn contact_pair(
        &self,
        handle1: CollisionObjectHandle,
        handle2: CollisionObjectHandle,
    ) -> Option<&ContactAlgorithm<N>> {
        let key = SortedPair::new(handle1, handle2);
        self.contact_generators.get(&key)
    }

    fn contact_pairs<'a>(
        &'a self,
        objects: &'a CollisionObjectSlab<N, T>,
    ) -> ContactPairs<'a, N, T> {
        ContactPairs::new(objects, self.contact_generators.iter())
    }

    fn proximity_pairs<'a>(
        &'a self,
        objects: &'a CollisionObjectSlab<N, T>,
    ) -> ProximityPairs<'a, N, T> {
        ProximityPairs::new(objects, self.proximity_detectors.iter())
    }
}
<|endoftext|>


<commit_before>#[cfg(feature = "lazy-blob")]
compile_error!("Tests must be run without lazy blobs.");

use libtest_mimic::{run_tests, Arguments, Outcome, Test};
use std::error::Error;
use std::fs::{read_dir, read_to_string};
use std::path::Path;
use wasmbin::io::Decode;
use wasmbin::module::Module;
use wast::parser::{parse, ParseBuffer};
use wast::Wast;

struct WasmTest {
    module: Vec<u8>,
    expect_result: Result<(), String>,
}

fn read_tests(path: &Path, dest: &mut Vec<Test<WasmTest>>) -> Result<(), Box<dyn Error>> {
    let src = read_to_string(path)?;
    let set_err_path_text = |mut err: wast::Error| {
        err.set_path(path);
        err.set_text(&src);
        err
    };
    let buf = ParseBuffer::new(&src).map_err(&set_err_path_text)?;
    let wast = parse::<Wast>(&buf).map_err(&set_err_path_text)?;
    for directive in wast.directives {
        let (span, mut module, expect_result) = match directive {
            wast::WastDirective::AssertMalformed {
                span,
                module: wast::QuoteModule::Module(module),
                message,
            } => (span, module, Err(message)),
            wast::WastDirective::Module(module) => (module.span, module, Ok(())),
            _ => {
                // Skipping interpreted
                continue;
            }
        };
        let (line, col) = span.linecol_in(&src);
        dest.push(Test {
            name: format!("{}:{}:{}", path.display(), line + 1, col + 1),
            kind: String::default(),
            is_ignored: match expect_result {
                // Our low-level parser doesn't validate sections.
                Err("function and code section have inconsistent lengths") => true,
                // We intentionally read future table/memory IDs.
                Err("zero flag expected") => true,
                _ => false,
            },
            is_bench: false,
            data: WasmTest {
                module: module.encode()?,
                expect_result: expect_result.map_err(|err| err.to_owned()),
            },
        });
    }
    Ok(())
}

fn main() -> Result<(), Box<dyn Error>> {
    let mut tests = Vec::new();
    for file in read_dir(Path::new("tests").join("testsuite"))? {
        let path = file?.path();
        if !path.extension().map_or(false, |ext| ext == "wast") {
            continue;
        }
        if let Err(err) = read_tests(&path, &mut tests) {
            // If an error comes from the Wabt, ignore for now - most likely it uses a new feature we don't support yet.
            eprintln!("Could not read test {}: {}", path.display(), err);
        }
    }
    assert!(
        !tests.is_empty(),
        "Couldn't find any tests. Did you run `git submodule update --init`?"
    );
    run_tests(&Arguments::from_args(), tests, |test| {
        match (
            Module::decode(&mut test.data.module.as_slice()),
            &test.data.expect_result,
        ) {
            (Ok(_), Err(err)) => Outcome::Failed {
                msg: Some(format!(
                    "Expected an invalid module definition with an error: {}",
                    err
                )),
            },
            (Err(err), Ok(())) => Outcome::Failed {
                msg: Some(format!(
                    "Expected a valid module definition, but got an error:\n{:#}",
                    err
                )),
            },
            _ => Outcome::Passed,
        }
    })
    .exit()
}
<commit_msg>Improve spec tests<commit_after>#[cfg(feature = "lazy-blob")]
compile_error!("Tests must be run without lazy blobs.");

use libtest_mimic::{run_tests, Arguments, Outcome, Test};
use std::error::Error;
use std::fs::{read_dir, read_to_string};
use std::path::Path;
use wasmbin::io::{Decode, Encode};
use wasmbin::module::Module;
use wast::parser::{parse, ParseBuffer};
use wast::Wast;

const IGNORED_ERRORS: &[&str] = &[
    // We don't perform cross-section analysis.
    "function and code section have inconsistent lengths",
    // This error is actually from a test that checks for duplicate section.
    // Same as above, we don't perform cross-section analysis.
    "junk after last section",
    // We allow non-zero table and memory IDs already.
    "zero flag expected",
];

struct WasmTest {
    module: Vec<u8>,
    expect_result: Result<(), String>,
}

fn read_tests(path: &Path, dest: &mut Vec<Test<WasmTest>>) -> Result<(), Box<dyn Error>> {
    let src = read_to_string(path)?;
    let set_err_path_text = |mut err: wast::Error| {
        err.set_path(path);
        err.set_text(&src);
        err
    };
    let buf = ParseBuffer::new(&src).map_err(&set_err_path_text)?;
    let wast = parse::<Wast>(&buf).map_err(&set_err_path_text)?;
    for directive in wast.directives {
        let (span, mut module, expect_result) = match directive {
            wast::WastDirective::AssertMalformed {
                span,
                module: wast::QuoteModule::Module(module),
                message,
            } => (span, module, Err(message)),
            wast::WastDirective::Module(module) => (module.span, module, Ok(())),
            _ => {
                // Skipping interpreted
                continue;
            }
        };
        let (line, col) = span.linecol_in(&src);
        dest.push(Test {
            name: format!("{}:{}:{}", path.display(), line + 1, col + 1),
            kind: String::default(),
            is_ignored: match expect_result {
                Ok(()) => false,
                Err(err) => IGNORED_ERRORS.contains(&err),
            },
            is_bench: false,
            data: WasmTest {
                module: module.encode()?,
                expect_result: expect_result.map_err(|err| err.to_owned()),
            },
        });
    }
    Ok(())
}

fn run_test(test: &WasmTest) -> Result<(), Box<dyn Error>> {
    let module = match (
        Module::decode(&mut test.module.as_slice()),
        &test.expect_result,
    ) {
        (Ok(_), Err(err)) => {
            return Err(format!(
                "Expected an invalid module definition with an error: {}",
                err
            )
            .into());
        }
        (Err(err), Ok(())) => {
            return Err(format!(
                "Expected a valid module definition, but got an error:\n{:#}",
                err
            )
            .into());
        }
        (Ok(module), Ok(())) => module,
        (Err(_), Err(_)) => return Ok(()),
    };
    let mut out = Vec::new();
    module.encode(&mut out)?;
    if out != test.module {
        // In the rare case that binary representation doesn't match, it
        // might be because the test uses longer LEB128 form than
        // required. Verify that at least decoding it back produces the
        // same module.
        let module2 = Module::decode(&mut out.as_slice())?;
        if module != module2 {
            return Err(format!(
                "Roundtrip mismatch. Old: {:#?}\nNew: {:#?}",
                module, module2
            )
            .into());
        }
    }
    Ok(())
}

fn main() -> Result<(), Box<dyn Error>> {
    let mut tests = Vec::new();
    for file in read_dir(Path::new("tests").join("testsuite"))? {
        let path = file?.path();
        if !path.extension().map_or(false, |ext| ext == "wast") {
            continue;
        }
        if let Err(err) = read_tests(&path, &mut tests) {
            // If an error comes from the Wabt, ignore for now - most likely it uses a new feature we don't support yet.
            eprintln!("Could not read test {}: {}", path.display(), err);
        }
    }
    assert!(
        !tests.is_empty(),
        "Couldn't find any tests. Did you run `git submodule update --init`?"
    );
    run_tests(&Arguments::from_args(), tests, |test| {
        match run_test(&test.data) {
            Ok(()) => Outcome::Passed,
            Err(err) => Outcome::Failed {
                msg: Some(err.to_string()),
            },
        }
    })
    .exit()
}
<|endoftext|>


<commit_before>// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

extern crate gcc;
extern crate cmake;
extern crate pkg_config;

const GRPC_VERSION: &'static str = "1.3.0";

#[cfg(not(feature = "static-link"))]
mod imp {
    use super::GRPC_VERSION;

    pub fn build_or_link_grpc(cc: &mut ::gcc::Config) {
        if let Ok(lib) = ::pkg_config::Config::new()
               .atleast_version(GRPC_VERSION)
               .probe("grpc_unsecure") {
            for inc_path in lib.include_paths {
                cc.include(inc_path);
            }
        } else {
            panic!("can't find a dynamic grpc library");
        }
    }
}

#[cfg(feature = "static-link")]
mod imp {
    use std::env;
    use std::fs::DirBuilder;
    use std::process::Command;

    use super::GRPC_VERSION;

    const ZLIB_VERSION: &'static str = "1.2.8";
    const BORINGSSL_COMMIT_HASH: &'static str = "78684e5b222645828ca302e56b40b9daff2b2d27";
    const CARES_TAG: &'static str = "cares-1_12_0";

    struct FakeGitSubmodule {
        url: String,
        base_name: String,
        to_dir: String,
    }

    impl FakeGitSubmodule {
        fn new_with_hash(account: &str,
                         repo: &str,
                         commit_hash: &str,
                         to_dir: &str)
                         -> FakeGitSubmodule {
            FakeGitSubmodule {
                url: format!("https://github.com/{}/{}/archive/{}.tar.gz",
                             account,
                             repo,
                             commit_hash),
                base_name: format!("{}-{}", repo, commit_hash),
                to_dir: to_dir.to_owned(),
            }
        }

        fn new_with_tag(account: &str, repo: &str, tag: &str, to_dir: &str) -> FakeGitSubmodule {
            FakeGitSubmodule {
                url: format!("https://github.com/{}/{}/archive/{}.tar.gz",
                             account,
                             repo,
                             tag),
                base_name: format!("{}-{}", repo, tag),
                to_dir: to_dir.to_owned(),
            }
        }

        fn new_with_semver(account: &str, repo: &str, ver: &str, to_dir: &str) -> FakeGitSubmodule {
            FakeGitSubmodule {
                url: format!("https://github.com/{}/{}/archive/v{}.tar.gz",
                             account,
                             repo,
                             ver),
                base_name: format!("{}-{}", repo, ver),
                to_dir: to_dir.to_owned(),
            }
        }

        fn init(&self) -> Result<(), String> {
            let out_dir = env::var_os("OUT_DIR").unwrap();
            let tgz_file_name = format!("{}.tar.gz", self.base_name);
            try!(Command::new("wget")
                     .args(&["-q", "-c", "-O", &tgz_file_name, &self.url])
                     .current_dir(&out_dir)
                     .status()
                     .map_err(|err| format!("failed to execute wget: {}", err))
                     .and_then(|status| if status.success() {
                                   Ok(())
                               } else {
                                   Err(format!("wget exit with {}", status))
                               }));

            try!(Command::new("tar")
                     .args(&["zxf", &tgz_file_name])
                     .current_dir(&out_dir)
                     .status()
                     .map_err(|err| format!("failed to execute tar: {}", err))
                     .and_then(|status| if status.success() {
                                   Ok(())
                               } else {
                                   Err(format!("tar exit with {}", status))
                               }));
            // clean base dir
            try!(Command::new("rm")
                     .args(&["-r", &self.to_dir])
                     .current_dir(&out_dir)
                     .status()
                     .map_err(|err| format!("failed to execute tar: {}", err)));

            Command::new("mv")
                .args(&[&self.base_name, &self.to_dir])
                .current_dir(&out_dir)
                .status()
                .map_err(|err| format!("mv execute failed: {}", err))
                .and_then(|status| if status.success() {
                              Ok(())
                          } else {
                              Err(format!("mv exit with {}", status))
                          })
        }
    }

    pub fn build_or_link_grpc(cc: &mut ::gcc::Config) {
        let out_dir = env::var("OUT_DIR").expect("Can't access OUT_DIR");
        DirBuilder::new()
            .recursive(true)
            .create(format!("{}/{}", out_dir, "grpc"))
            .unwrap();

        FakeGitSubmodule::new_with_semver("grpc", "grpc", GRPC_VERSION, "grpc")
            .init()
            .unwrap();

        FakeGitSubmodule::new_with_semver("madler", "zlib", ZLIB_VERSION, "grpc/third_party/zlib")
            .init()
            .unwrap();

        FakeGitSubmodule::new_with_hash("google",
                                        "boringssl",
                                        BORINGSSL_COMMIT_HASH,
                                        "grpc/third_party/boringssl")
                .init()
                .unwrap();

        FakeGitSubmodule::new_with_tag("c-ares",
                                       "c-ares",
                                       CARES_TAG,
                                       "grpc/third_party/cares/cares")
                .init()
                .unwrap();

        // fix multiple _main symbols
        let _ = Command::new("rm")
            .current_dir(&format!("{}/grpc/third_party/cares/cares", out_dir))
            .args(&["acountry.c", "adig.c", "ahost.c"])
            .status();

        let dst = ::cmake::Config::new(format!("{}/grpc", out_dir))
            .build_target("grpc")
            .build();

        println!("cargo:rustc-link-search=native={}/build", dst.display());
        println!("cargo:rustc-link-search=native={}/build/third_party/cares",
                 dst.display());
        println!("cargo:rustc-link-search=native={}/build/third_party/zlib",
                 dst.display());

        println!("cargo:rustc-link-lib=static=z");
        println!("cargo:rustc-link-lib=static=cares");
        println!("cargo:rustc-link-lib=static=gpr");
        println!("cargo:rustc-link-lib=static=grpc");

        cc.include(format!("{}/grpc/include", out_dir));
    }
}

fn main() {
    let mut cc = ::gcc::Config::new();

    imp::build_or_link_grpc(&mut cc);

    cc.file("grpc_wrap.c")
        .flag("-fPIC")
        .flag("-O2")
        .compile("libgrpc_wrap.a");
}
<commit_msg>build.rs: now static build at OUT_DIR<commit_after>// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

extern crate gcc;
extern crate cmake;
extern crate pkg_config;

const GRPC_VERSION: &'static str = "1.3.0";

#[cfg(not(feature = "static-link"))]
mod imp {
    use super::GRPC_VERSION;

    pub fn build_or_link_grpc(cc: &mut ::gcc::Config) {
        if let Ok(lib) = ::pkg_config::Config::new()
               .atleast_version(GRPC_VERSION)
               .probe("grpc_unsecure") {
            for inc_path in lib.include_paths {
                cc.include(inc_path);
            }
        } else {
            panic!("can't find a dynamic grpc library");
        }
    }
}

#[cfg(feature = "static-link")]
mod imp {
    use std::env;
    use std::fs::DirBuilder;
    use std::process::Command;

    use super::GRPC_VERSION;

    const ZLIB_VERSION: &'static str = "1.2.8";
    const BORINGSSL_COMMIT_HASH: &'static str = "78684e5b222645828ca302e56b40b9daff2b2d27";
    const CARES_TAG: &'static str = "cares-1_12_0";

    struct FakeGitSubmodule {
        url: String,
        base_name: String,
        to_dir: String,
    }

    impl FakeGitSubmodule {
        fn new_with_hash(account: &str,
                         repo: &str,
                         commit_hash: &str,
                         to_dir: &str)
                         -> FakeGitSubmodule {
            FakeGitSubmodule {
                url: format!("https://github.com/{}/{}/archive/{}.tar.gz",
                             account,
                             repo,
                             commit_hash),
                base_name: format!("{}-{}", repo, commit_hash),
                to_dir: to_dir.to_owned(),
            }
        }

        fn new_with_tag(account: &str, repo: &str, tag: &str, to_dir: &str) -> FakeGitSubmodule {
            FakeGitSubmodule {
                url: format!("https://github.com/{}/{}/archive/{}.tar.gz",
                             account,
                             repo,
                             tag),
                base_name: format!("{}-{}", repo, tag),
                to_dir: to_dir.to_owned(),
            }
        }

        fn new_with_semver(account: &str, repo: &str, ver: &str, to_dir: &str) -> FakeGitSubmodule {
            FakeGitSubmodule {
                url: format!("https://github.com/{}/{}/archive/v{}.tar.gz",
                             account,
                             repo,
                             ver),
                base_name: format!("{}-{}", repo, ver),
                to_dir: to_dir.to_owned(),
            }
        }

        fn init(&self) -> Result<(), String> {
            let out_dir = env::var_os("OUT_DIR").unwrap();
            let tgz_file_name = format!("{}.tar.gz", self.base_name);
            try!(Command::new("wget")
                     .args(&["-q", "-c", "-O", &tgz_file_name, &self.url])
                     .current_dir(&out_dir)
                     .status()
                     .map_err(|err| format!("failed to execute wget: {}", err))
                     .and_then(|status| if status.success() {
                                   Ok(())
                               } else {
                                   Err(format!("wget exit with {}", status))
                               }));

            try!(Command::new("tar")
                     .args(&["zxf", &tgz_file_name])
                     .current_dir(&out_dir)
                     .status()
                     .map_err(|err| format!("failed to execute tar: {}", err))
                     .and_then(|status| if status.success() {
                                   Ok(())
                               } else {
                                   Err(format!("tar exit with {}", status))
                               }));
            // clean base dir
            try!(Command::new("rm")
                     .args(&["-r", &self.to_dir])
                     .current_dir(&out_dir)
                     .status()
                     .map_err(|err| format!("failed to execute tar: {}", err)));

            Command::new("mv")
                .args(&[&self.base_name, &self.to_dir])
                .current_dir(&out_dir)
                .status()
                .map_err(|err| format!("mv execute failed: {}", err))
                .and_then(|status| if status.success() {
                              Ok(())
                          } else {
                              Err(format!("mv exit with {}", status))
                          })
        }
    }

    pub fn build_or_link_grpc(cc: &mut gcc::Config) {
        let out_dir = env::var("OUT_DIR").expect("Can't access OUT_DIR");
        DirBuilder::new()
            .recursive(true)
            .create(format!("{}/{}", out_dir, "grpc"))
            .unwrap();

        FakeGitSubmodule::new_with_semver("grpc", "grpc", GRPC_VERSION, "grpc")
            .init()
            .unwrap();

        FakeGitSubmodule::new_with_semver("madler", "zlib", ZLIB_VERSION, "grpc/third_party/zlib")
            .init()
            .unwrap();

        FakeGitSubmodule::new_with_hash("google",
                                        "boringssl",
                                        BORINGSSL_COMMIT_HASH,
                                        "grpc/third_party/boringssl")
                .init()
                .unwrap();

        FakeGitSubmodule::new_with_tag("c-ares",
                                       "c-ares",
                                       CARES_TAG,
                                       "grpc/third_party/cares/cares")
                .init()
                .unwrap();

        // fix multiple _main symbols
        let _ = Command::new("rm")
            .current_dir(&format!("{}/grpc/third_party/cares/cares", out_dir))
            .args(&["acountry.c", "adig.c", "ahost.c"])
            .status();

        let dst = cmake::Config::new(format!("{}/grpc", out_dir))
            .build_target("grpc")
            .build();

        println!("cargo:rustc-link-search=native={}/build", dst.display());
        println!("cargo:rustc-link-search=native={}/build/third_party/cares",
                 dst.display());
        println!("cargo:rustc-link-search=native={}/build/third_party/zlib",
                 dst.display());

        println!("cargo:rustc-link-lib=static=z");
        println!("cargo:rustc-link-lib=static=cares");
        println!("cargo:rustc-link-lib=static=gpr");
        println!("cargo:rustc-link-lib=static=grpc");

        cc.include(format!("{}/grpc/include", out_dir));
    }
}

fn main() {
    let mut cc = gcc::Config::new();

    imp::build_or_link_grpc(&mut cc);

    cc.file("grpc_wrap.c")
        .flag("-fPIC")
        .flag("-O2")
        .compile("libgrpc_wrap.a");
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use eutil::fptr_is_null;
use libc::{c_int};
use std::mem;
use string::{cef_string_userfree_utf8_alloc,cef_string_userfree_utf8_free,cef_string_utf8_set};
use types::{cef_string_list_t,cef_string_t};

//cef_string_list

#[no_mangle]
extern "C" fn cef_string_list_alloc() -> *mut cef_string_list_t {
    unsafe {
         let lt: Box<Vec<*mut cef_string_t>> = box vec!();
         mem::transmute(lt)
    }
}

#[no_mangle]
extern "C" fn cef_string_list_size(lt: *const cef_string_list_t) -> c_int {
    unsafe {
        if fptr_is_null(mem::transmute(lt)) { return 0; }
        let v: Box<Vec<*mut cef_string_t>> = mem::transmute(lt);
        v.len() as c_int
    }
}
<commit_msg>embedding: cef_string_list_append()<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use eutil::fptr_is_null;
use libc::{c_int};
use std::mem;
use string::{cef_string_userfree_utf8_alloc,cef_string_userfree_utf8_free,cef_string_utf8_set};
use types::{cef_string_list_t,cef_string_t};

//cef_string_list

#[no_mangle]
extern "C" fn cef_string_list_alloc() -> *mut cef_string_list_t {
    unsafe {
         let lt: Box<Vec<*mut cef_string_t>> = box vec!();
         mem::transmute(lt)
    }
}

#[no_mangle]
extern "C" fn cef_string_list_size(lt: *const cef_string_list_t) -> c_int {
    unsafe {
        if fptr_is_null(mem::transmute(lt)) { return 0; }
        let v: Box<Vec<*mut cef_string_t>> = mem::transmute(lt);
        v.len() as c_int
    }
}

#[no_mangle]
extern "C" fn cef_string_list_append(lt: *mut cef_string_list_t, value: *const cef_string_t) {
    unsafe {
        if fptr_is_null(mem::transmute(lt)) { return; }
        let mut v: Box<Vec<*mut cef_string_t>> = mem::transmute(lt);
        let cs = cef_string_userfree_utf8_alloc();
        cef_string_utf8_set(mem::transmute((*value).str), (*value).length, cs, 1);
        v.push(cs);
    }
}
<|endoftext|>


<commit_before>// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0.

use std::sync::Arc;
use std::time::Instant;

use engine::DB;
use engine_rocks::{RocksEngine, RocksSstWriter, RocksSstWriterBuilder};
use engine_traits::{CfName, CF_DEFAULT, CF_WRITE};
use engine_traits::{ExternalSstFileInfo, SstWriter, SstWriterBuilder};
use external_storage::ExternalStorage;
use futures_util::io::AllowStdIo;
use kvproto::backup::File;
use tikv::coprocessor::checksum_crc64_xor;
use tikv::storage::txn::TxnEntry;
use tikv_util::{self, box_err, file::Sha256Reader, time::Limiter};
use txn_types::KvPair;

use crate::metrics::*;
use crate::{Error, Result};

struct Writer {
    writer: RocksSstWriter,
    total_kvs: u64,
    total_bytes: u64,
    checksum: u64,
    digest: crc64fast::Digest,
}

impl Writer {
    fn new(writer: RocksSstWriter) -> Self {
        Writer {
            writer,
            total_kvs: 0,
            total_bytes: 0,
            checksum: 0,
            digest: crc64fast::Digest::new(),
        }
    }

    fn write(&mut self, key: &[u8], value: &[u8]) -> Result<()> {
        // HACK: The actual key stored in TiKV is called
        // data_key and always prefix a `z`. But iterator strips
        // it, we need to add the prefix manually.
        let data_key_write = keys::data_key(key);
        self.writer.put(&data_key_write, value)?;
        Ok(())
    }

    fn update_with(&mut self, entry: TxnEntry, need_checksum: bool) -> Result<()> {
        self.total_kvs += 1;
        if need_checksum {
            let (k, v) = entry
                .into_kvpair()
                .map_err(|err| Error::Other(box_err!("Decode error: {:?}", err)))?;
            self.total_bytes += (k.len() + v.len()) as u64;
            self.checksum = checksum_crc64_xor(self.checksum, self.digest.clone(), &k, &v);
        }
        Ok(())
    }

    fn update_raw_with(&mut self, key: &[u8], value: &[u8], need_checksum: bool) -> Result<()> {
        self.total_kvs += 1;
        self.total_bytes += (key.len() + value.len()) as u64;
        if need_checksum {
            self.checksum = checksum_crc64_xor(self.checksum, self.digest.clone(), key, value);
        }
        Ok(())
    }

    fn save_and_build_file(
        self,
        name: &str,
        cf: &'static str,
        limiter: Limiter,
        storage: &dyn ExternalStorage,
    ) -> Result<File> {
        let (sst_info, sst_reader) = self.writer.finish_read()?;
        BACKUP_RANGE_SIZE_HISTOGRAM_VEC
            .with_label_values(&[cf])
            .observe(sst_info.file_size() as f64);
        let file_name = format!("{}_{}.sst", name, cf);

        let (reader, hasher) = Sha256Reader::new(sst_reader)
            .map_err(|e| Error::Other(box_err!("Sha256 error: {:?}", e)))?;
        storage.write(
            &file_name,
            Box::new(limiter.limit(AllowStdIo::new(reader))),
            sst_info.file_size(),
        )?;
        let sha256 = hasher
            .lock()
            .unwrap()
            .finish()
            .map(|digest| digest.to_vec())
            .map_err(|e| Error::Other(box_err!("Sha256 error: {:?}", e)))?;

        let mut file = File::default();
        file.set_name(file_name);
        file.set_sha256(sha256);
        file.set_crc64xor(self.checksum);
        file.set_total_kvs(self.total_kvs);
        file.set_total_bytes(self.total_bytes);
        file.set_cf(cf.to_owned());
        Ok(file)
    }

    fn is_empty(&self) -> bool {
        self.total_kvs == 0
    }
}

/// A writer writes txn entries into SST files.
pub struct BackupWriter {
    name: String,
    default: Writer,
    write: Writer,
    limiter: Limiter,
}

impl BackupWriter {
    /// Create a new BackupWriter.
    pub fn new(db: Arc<DB>, name: &str, limiter: Limiter) -> Result<BackupWriter> {
        let default = RocksSstWriterBuilder::new()
            .set_in_memory(true)
            .set_cf(CF_DEFAULT)
            .set_db(RocksEngine::from_ref(&db))
            .build(name)?;
        let write = RocksSstWriterBuilder::new()
            .set_in_memory(true)
            .set_cf(CF_WRITE)
            .set_db(RocksEngine::from_ref(&db))
            .build(name)?;
        let name = name.to_owned();
        Ok(BackupWriter {
            name,
            default: Writer::new(default),
            write: Writer::new(write),
            limiter,
        })
    }

    /// Write entries to buffered SST files.
    pub fn write<I>(&mut self, entries: I, need_checksum: bool) -> Result<()>
    where
        I: Iterator<Item = TxnEntry>,
    {
        for e in entries {
            let mut value_in_default = false;
            match &e {
                TxnEntry::Commit { default, write } => {
                    // Default may be empty if value is small.
                    if !default.0.is_empty() {
                        self.default.write(&default.0, &default.1)?;
                        value_in_default = true;
                    }
                    assert!(!write.0.is_empty());
                    self.write.write(&write.0, &write.1)?;
                }
                TxnEntry::Prewrite { .. } => {
                    return Err(Error::Other("prewrite is not supported".into()));
                }
            }
            if value_in_default {
                self.default.update_with(e, need_checksum)?;
            } else {
                self.write.update_with(e, need_checksum)?;
            }
        }
        Ok(())
    }

    /// Save buffered SST files to the given external storage.
    pub fn save(self, storage: &dyn ExternalStorage) -> Result<Vec<File>> {
        let start = Instant::now();
        let mut files = Vec::with_capacity(2);
        let write_written = !self.write.is_empty() || !self.default.is_empty();
        if !self.default.is_empty() {
            // Save default cf contents.
            let default = self.default.save_and_build_file(
                &self.name,
                CF_DEFAULT,
                self.limiter.clone(),
                storage,
            )?;
            files.push(default);
        }
        if write_written {
            // Save write cf contents.
            let write = self.write.save_and_build_file(
                &self.name,
                CF_WRITE,
                self.limiter.clone(),
                storage,
            )?;
            files.push(write);
        }
        BACKUP_RANGE_HISTOGRAM_VEC
            .with_label_values(&["save"])
            .observe(start.elapsed().as_secs_f64());
        Ok(files)
    }
}

/// A writer writes Raw kv into SST files.
pub struct BackupRawKVWriter {
    name: String,
    cf: CfName,
    writer: Writer,
    limiter: Limiter,
}

impl BackupRawKVWriter {
    /// Create a new BackupRawKVWriter.
    pub fn new(db: Arc<DB>, name: &str, cf: CfName, limiter: Limiter) -> Result<BackupRawKVWriter> {
        let writer = RocksSstWriterBuilder::new()
            .set_in_memory(true)
            .set_cf(cf)
            .set_db(RocksEngine::from_ref(&db))
            .build(name)?;
        Ok(BackupRawKVWriter {
            name: name.to_owned(),
            cf,
            writer: Writer::new(writer),
            limiter,
        })
    }

    /// Write Kv_pair to buffered SST files.
    pub fn write<I>(&mut self, kv_pairs: I, need_checksum: bool) -> Result<()>
    where
        I: Iterator<Item = Result<KvPair>>,
    {
        for kv_pair in kv_pairs {
            let (k, v) = match kv_pair {
                Ok(s) => s,
                Err(e) => {
                    error!("write raw kv"; "error" => ?e);
                    return Err(Error::Other("occur an error when written raw kv".into()));
                }
            };

            assert!(!k.is_empty());
            self.writer.write(&k, &v)?;
            self.writer.update_raw_with(&k, &v, need_checksum)?;
        }
        Ok(())
    }

    /// Save buffered SST files to the given external storage.
    pub fn save(self, storage: &dyn ExternalStorage) -> Result<Vec<File>> {
        let start = Instant::now();
        let mut files = Vec::with_capacity(1);
        if !self.writer.is_empty() {
            let file = self.writer.save_and_build_file(
                &self.name,
                self.cf,
                self.limiter.clone(),
                storage,
            )?;
            files.push(file);
        }
        BACKUP_RANGE_HISTOGRAM_VEC
            .with_label_values(&["save_raw"])
            .observe(start.elapsed().as_secs_f64());
        Ok(files)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use engine::Iterable;
    use std::collections::BTreeMap;
    use std::f64::INFINITY;
    use std::path::Path;
    use tempfile::TempDir;
    use tikv::storage::TestEngineBuilder;

    type CfKvs<'a> = (engine_traits::CfName, &'a [(&'a [u8], &'a [u8])]);

    fn check_sst(ssts: &[(engine_traits::CfName, &Path)], kvs: &[CfKvs]) {
        let temp = TempDir::new().unwrap();
        let rocks = TestEngineBuilder::new()
            .path(temp.path())
            .cfs(&[engine_traits::CF_DEFAULT, engine_traits::CF_WRITE])
            .build()
            .unwrap();
        let db = rocks.get_rocksdb();

        let opt = engine::rocks::IngestExternalFileOptions::new();
        for (cf, sst) in ssts {
            let handle = db.cf_handle(cf).unwrap();
            db.ingest_external_file_cf(handle, &opt, &[sst.to_str().unwrap()])
                .unwrap();
        }
        for (cf, kv) in kvs {
            let mut map = BTreeMap::new();
            db.scan_cf(
                cf,
                keys::DATA_MIN_KEY,
                keys::DATA_MAX_KEY,
                false,
                |key, value| {
                    map.insert(key.to_owned(), value.to_owned());
                    Ok(true)
                },
            )
            .unwrap();
            assert_eq!(map.len(), kv.len(), "{:?} {:?}", map, kv);
            for (k, v) in *kv {
                assert_eq!(&v.to_vec(), map.get(&k.to_vec()).unwrap());
            }
        }
    }

    #[test]
    fn test_writer() {
        let temp = TempDir::new().unwrap();
        let rocks = TestEngineBuilder::new()
            .path(temp.path())
            .cfs(&[
                engine_traits::CF_DEFAULT,
                engine_traits::CF_LOCK,
                engine_traits::CF_WRITE,
            ])
            .build()
            .unwrap();
        let db = rocks.get_rocksdb();
        let backend = external_storage::make_local_backend(temp.path());
        let storage = external_storage::create_storage(&backend).unwrap();

        // Test empty file.
        let mut writer = BackupWriter::new(db.clone(), "foo", Limiter::new(INFINITY)).unwrap();
        writer.write(vec![].into_iter(), false).unwrap();
        assert!(writer.save(&storage).unwrap().is_empty());

        // Test write only txn.
        let mut writer = BackupWriter::new(db.clone(), "foo1", Limiter::new(INFINITY)).unwrap();
        writer
            .write(
                vec![TxnEntry::Commit {
                    default: (vec![], vec![]),
                    write: (vec![b'a'], vec![b'a']),
                }]
                .into_iter(),
                false,
            )
            .unwrap();
        let files = writer.save(&storage).unwrap();
        assert_eq!(files.len(), 1);
        check_sst(
            &[(
                engine_traits::CF_WRITE,
                &temp.path().join(files[0].get_name()),
            )],
            &[(
                engine_traits::CF_WRITE,
                &[(&keys::data_key(&[b'a']), &[b'a'])],
            )],
        );

        // Test write and default.
        let mut writer = BackupWriter::new(db, "foo2", Limiter::new(INFINITY)).unwrap();
        writer
            .write(
                vec![
                    TxnEntry::Commit {
                        default: (vec![b'a'], vec![b'a']),
                        write: (vec![b'a'], vec![b'a']),
                    },
                    TxnEntry::Commit {
                        default: (vec![], vec![]),
                        write: (vec![b'b'], vec![]),
                    },
                ]
                .into_iter(),
                false,
            )
            .unwrap();
        let files = writer.save(&storage).unwrap();
        assert_eq!(files.len(), 2);
        check_sst(
            &[
                (
                    engine_traits::CF_DEFAULT,
                    &temp.path().join(files[0].get_name()),
                ),
                (
                    engine_traits::CF_WRITE,
                    &temp.path().join(files[1].get_name()),
                ),
            ],
            &[
                (
                    engine_traits::CF_DEFAULT,
                    &[
                        (&keys::data_key(&[b'a']), &[b'a']),
                        (&keys::data_key(&[]), &[]),
                    ],
                ),
                (
                    engine_traits::CF_WRITE,
                    &[
                        (&keys::data_key(&[b'a']), &[b'a']),
                        (&keys::data_key(&[b'b']), &[]),
                    ],
                ),
            ],
        );
    }
}
<commit_msg>backup: fix test_writer (#6970)<commit_after>// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0.

use std::sync::Arc;
use std::time::Instant;

use engine::DB;
use engine_rocks::{RocksEngine, RocksSstWriter, RocksSstWriterBuilder};
use engine_traits::{CfName, CF_DEFAULT, CF_WRITE};
use engine_traits::{ExternalSstFileInfo, SstWriter, SstWriterBuilder};
use external_storage::ExternalStorage;
use futures_util::io::AllowStdIo;
use kvproto::backup::File;
use tikv::coprocessor::checksum_crc64_xor;
use tikv::storage::txn::TxnEntry;
use tikv_util::{self, box_err, file::Sha256Reader, time::Limiter};
use txn_types::KvPair;

use crate::metrics::*;
use crate::{Error, Result};

struct Writer {
    writer: RocksSstWriter,
    total_kvs: u64,
    total_bytes: u64,
    checksum: u64,
    digest: crc64fast::Digest,
}

impl Writer {
    fn new(writer: RocksSstWriter) -> Self {
        Writer {
            writer,
            total_kvs: 0,
            total_bytes: 0,
            checksum: 0,
            digest: crc64fast::Digest::new(),
        }
    }

    fn write(&mut self, key: &[u8], value: &[u8]) -> Result<()> {
        // HACK: The actual key stored in TiKV is called
        // data_key and always prefix a `z`. But iterator strips
        // it, we need to add the prefix manually.
        let data_key_write = keys::data_key(key);
        self.writer.put(&data_key_write, value)?;
        Ok(())
    }

    fn update_with(&mut self, entry: TxnEntry, need_checksum: bool) -> Result<()> {
        self.total_kvs += 1;
        if need_checksum {
            let (k, v) = entry
                .into_kvpair()
                .map_err(|err| Error::Other(box_err!("Decode error: {:?}", err)))?;
            self.total_bytes += (k.len() + v.len()) as u64;
            self.checksum = checksum_crc64_xor(self.checksum, self.digest.clone(), &k, &v);
        }
        Ok(())
    }

    fn update_raw_with(&mut self, key: &[u8], value: &[u8], need_checksum: bool) -> Result<()> {
        self.total_kvs += 1;
        self.total_bytes += (key.len() + value.len()) as u64;
        if need_checksum {
            self.checksum = checksum_crc64_xor(self.checksum, self.digest.clone(), key, value);
        }
        Ok(())
    }

    fn save_and_build_file(
        self,
        name: &str,
        cf: &'static str,
        limiter: Limiter,
        storage: &dyn ExternalStorage,
    ) -> Result<File> {
        let (sst_info, sst_reader) = self.writer.finish_read()?;
        BACKUP_RANGE_SIZE_HISTOGRAM_VEC
            .with_label_values(&[cf])
            .observe(sst_info.file_size() as f64);
        let file_name = format!("{}_{}.sst", name, cf);

        let (reader, hasher) = Sha256Reader::new(sst_reader)
            .map_err(|e| Error::Other(box_err!("Sha256 error: {:?}", e)))?;
        storage.write(
            &file_name,
            Box::new(limiter.limit(AllowStdIo::new(reader))),
            sst_info.file_size(),
        )?;
        let sha256 = hasher
            .lock()
            .unwrap()
            .finish()
            .map(|digest| digest.to_vec())
            .map_err(|e| Error::Other(box_err!("Sha256 error: {:?}", e)))?;

        let mut file = File::default();
        file.set_name(file_name);
        file.set_sha256(sha256);
        file.set_crc64xor(self.checksum);
        file.set_total_kvs(self.total_kvs);
        file.set_total_bytes(self.total_bytes);
        file.set_cf(cf.to_owned());
        Ok(file)
    }

    fn is_empty(&self) -> bool {
        self.total_kvs == 0
    }
}

/// A writer writes txn entries into SST files.
pub struct BackupWriter {
    name: String,
    default: Writer,
    write: Writer,
    limiter: Limiter,
}

impl BackupWriter {
    /// Create a new BackupWriter.
    pub fn new(db: Arc<DB>, name: &str, limiter: Limiter) -> Result<BackupWriter> {
        let default = RocksSstWriterBuilder::new()
            .set_in_memory(true)
            .set_cf(CF_DEFAULT)
            .set_db(RocksEngine::from_ref(&db))
            .build(name)?;
        let write = RocksSstWriterBuilder::new()
            .set_in_memory(true)
            .set_cf(CF_WRITE)
            .set_db(RocksEngine::from_ref(&db))
            .build(name)?;
        let name = name.to_owned();
        Ok(BackupWriter {
            name,
            default: Writer::new(default),
            write: Writer::new(write),
            limiter,
        })
    }

    /// Write entries to buffered SST files.
    pub fn write<I>(&mut self, entries: I, need_checksum: bool) -> Result<()>
    where
        I: Iterator<Item = TxnEntry>,
    {
        for e in entries {
            let mut value_in_default = false;
            match &e {
                TxnEntry::Commit { default, write } => {
                    // Default may be empty if value is small.
                    if !default.0.is_empty() {
                        self.default.write(&default.0, &default.1)?;
                        value_in_default = true;
                    }
                    assert!(!write.0.is_empty());
                    self.write.write(&write.0, &write.1)?;
                }
                TxnEntry::Prewrite { .. } => {
                    return Err(Error::Other("prewrite is not supported".into()));
                }
            }
            if value_in_default {
                self.default.update_with(e, need_checksum)?;
            } else {
                self.write.update_with(e, need_checksum)?;
            }
        }
        Ok(())
    }

    /// Save buffered SST files to the given external storage.
    pub fn save(self, storage: &dyn ExternalStorage) -> Result<Vec<File>> {
        let start = Instant::now();
        let mut files = Vec::with_capacity(2);
        let write_written = !self.write.is_empty() || !self.default.is_empty();
        if !self.default.is_empty() {
            // Save default cf contents.
            let default = self.default.save_and_build_file(
                &self.name,
                CF_DEFAULT,
                self.limiter.clone(),
                storage,
            )?;
            files.push(default);
        }
        if write_written {
            // Save write cf contents.
            let write = self.write.save_and_build_file(
                &self.name,
                CF_WRITE,
                self.limiter.clone(),
                storage,
            )?;
            files.push(write);
        }
        BACKUP_RANGE_HISTOGRAM_VEC
            .with_label_values(&["save"])
            .observe(start.elapsed().as_secs_f64());
        Ok(files)
    }
}

/// A writer writes Raw kv into SST files.
pub struct BackupRawKVWriter {
    name: String,
    cf: CfName,
    writer: Writer,
    limiter: Limiter,
}

impl BackupRawKVWriter {
    /// Create a new BackupRawKVWriter.
    pub fn new(db: Arc<DB>, name: &str, cf: CfName, limiter: Limiter) -> Result<BackupRawKVWriter> {
        let writer = RocksSstWriterBuilder::new()
            .set_in_memory(true)
            .set_cf(cf)
            .set_db(RocksEngine::from_ref(&db))
            .build(name)?;
        Ok(BackupRawKVWriter {
            name: name.to_owned(),
            cf,
            writer: Writer::new(writer),
            limiter,
        })
    }

    /// Write Kv_pair to buffered SST files.
    pub fn write<I>(&mut self, kv_pairs: I, need_checksum: bool) -> Result<()>
    where
        I: Iterator<Item = Result<KvPair>>,
    {
        for kv_pair in kv_pairs {
            let (k, v) = match kv_pair {
                Ok(s) => s,
                Err(e) => {
                    error!("write raw kv"; "error" => ?e);
                    return Err(Error::Other("occur an error when written raw kv".into()));
                }
            };

            assert!(!k.is_empty());
            self.writer.write(&k, &v)?;
            self.writer.update_raw_with(&k, &v, need_checksum)?;
        }
        Ok(())
    }

    /// Save buffered SST files to the given external storage.
    pub fn save(self, storage: &dyn ExternalStorage) -> Result<Vec<File>> {
        let start = Instant::now();
        let mut files = Vec::with_capacity(1);
        if !self.writer.is_empty() {
            let file = self.writer.save_and_build_file(
                &self.name,
                self.cf,
                self.limiter.clone(),
                storage,
            )?;
            files.push(file);
        }
        BACKUP_RANGE_HISTOGRAM_VEC
            .with_label_values(&["save_raw"])
            .observe(start.elapsed().as_secs_f64());
        Ok(files)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use engine::Iterable;
    use std::collections::BTreeMap;
    use std::f64::INFINITY;
    use std::path::Path;
    use tempfile::TempDir;
    use tikv::storage::TestEngineBuilder;

    type CfKvs<'a> = (engine_traits::CfName, &'a [(&'a [u8], &'a [u8])]);

    fn check_sst(ssts: &[(engine_traits::CfName, &Path)], kvs: &[CfKvs]) {
        let temp = TempDir::new().unwrap();
        let rocks = TestEngineBuilder::new()
            .path(temp.path())
            .cfs(&[engine_traits::CF_DEFAULT, engine_traits::CF_WRITE])
            .build()
            .unwrap();
        let db = rocks.get_rocksdb();

        let opt = engine::rocks::IngestExternalFileOptions::new();
        for (cf, sst) in ssts {
            let handle = db.cf_handle(cf).unwrap();
            db.ingest_external_file_cf(handle, &opt, &[sst.to_str().unwrap()])
                .unwrap();
        }
        for (cf, kv) in kvs {
            let mut map = BTreeMap::new();
            db.scan_cf(
                cf,
                keys::DATA_MIN_KEY,
                keys::DATA_MAX_KEY,
                false,
                |key, value| {
                    map.insert(key.to_owned(), value.to_owned());
                    Ok(true)
                },
            )
            .unwrap();
            assert_eq!(map.len(), kv.len(), "{} {:?} {:?}", cf, map, kv);
            for (k, v) in *kv {
                assert_eq!(&v.to_vec(), map.get(&k.to_vec()).unwrap());
            }
        }
    }

    #[test]
    fn test_writer() {
        let temp = TempDir::new().unwrap();
        let rocks = TestEngineBuilder::new()
            .path(temp.path())
            .cfs(&[
                engine_traits::CF_DEFAULT,
                engine_traits::CF_LOCK,
                engine_traits::CF_WRITE,
            ])
            .build()
            .unwrap();
        let db = rocks.get_rocksdb();
        let backend = external_storage::make_local_backend(temp.path());
        let storage = external_storage::create_storage(&backend).unwrap();

        // Test empty file.
        let mut writer = BackupWriter::new(db.clone(), "foo", Limiter::new(INFINITY)).unwrap();
        writer.write(vec![].into_iter(), false).unwrap();
        assert!(writer.save(&storage).unwrap().is_empty());

        // Test write only txn.
        let mut writer = BackupWriter::new(db.clone(), "foo1", Limiter::new(INFINITY)).unwrap();
        writer
            .write(
                vec![TxnEntry::Commit {
                    default: (vec![], vec![]),
                    write: (vec![b'a'], vec![b'a']),
                }]
                .into_iter(),
                false,
            )
            .unwrap();
        let files = writer.save(&storage).unwrap();
        assert_eq!(files.len(), 1);
        check_sst(
            &[(
                engine_traits::CF_WRITE,
                &temp.path().join(files[0].get_name()),
            )],
            &[(
                engine_traits::CF_WRITE,
                &[(&keys::data_key(&[b'a']), &[b'a'])],
            )],
        );

        // Test write and default.
        let mut writer = BackupWriter::new(db, "foo2", Limiter::new(INFINITY)).unwrap();
        writer
            .write(
                vec![
                    TxnEntry::Commit {
                        default: (vec![b'a'], vec![b'a']),
                        write: (vec![b'a'], vec![b'a']),
                    },
                    TxnEntry::Commit {
                        default: (vec![], vec![]),
                        write: (vec![b'b'], vec![]),
                    },
                ]
                .into_iter(),
                false,
            )
            .unwrap();
        let files = writer.save(&storage).unwrap();
        assert_eq!(files.len(), 2);
        check_sst(
            &[
                (
                    engine_traits::CF_DEFAULT,
                    &temp.path().join(files[0].get_name()),
                ),
                (
                    engine_traits::CF_WRITE,
                    &temp.path().join(files[1].get_name()),
                ),
            ],
            &[
                (
                    engine_traits::CF_DEFAULT,
                    &[(&keys::data_key(&[b'a']), &[b'a'])],
                ),
                (
                    engine_traits::CF_WRITE,
                    &[
                        (&keys::data_key(&[b'a']), &[b'a']),
                        (&keys::data_key(&[b'b']), &[]),
                    ],
                ),
            ],
        );
    }
}
<|endoftext|>


<commit_before>use std::collections::HashMap;
use std::io::Read;

use hyper::status::StatusCode;
use rustc_serialize::json;
use url::{ParseError, Url};
use url::form_urlencoded::serialize_owned;

use error::Error;
use http;
use response::Response;

#[derive(Debug)]
pub struct Client {
    root_url: String,
}

impl Client {
    pub fn new(root_url: &str) -> Result<Client, ParseError> {
        let url = try!(Url::parse(root_url));
        let client = Client {
            root_url: format!("{}", url.serialize()),
        };

        Ok(client)
    }

    pub fn default() -> Client {
        Client {
            root_url: "http://127.0.0.1:2379/".to_string(),
        }
    }

    pub fn create(&self, key: &str, value: &str, ttl: Option<u64>) -> Result<Response, Error> {
        self.create_or_set(key, value, ttl, false)
    }

    pub fn delete(&self, key: &str, recursive: bool) -> Result<Response, Error> {
        let url = self.build_url(key);
        let mut options = vec![];

        options.push(("recursive".to_string(), format!("{}", recursive)));

        let body = serialize_owned(&options);

        let mut response = try!(http::delete(url, body));
        let mut response_body = String::new();
        response.read_to_string(&mut response_body).unwrap();

        match response.status {
            StatusCode::Ok => Ok(json::decode(&response_body).unwrap()),
            _ => Err(Error::Etcd(json::decode(&response_body).unwrap())),
        }
    }

    pub fn get(&self, key: &str, sort: bool, recursive: bool) -> Result<Response, Error> {
        let base_url = self.build_url(key);
        let sort_string = format!("{}", sort);
        let recursive_string = format!("{}", recursive);

        let mut query_pairs = HashMap::new();

        query_pairs.insert("sorted", &sort_string[..]);
        query_pairs.insert("recursive", &recursive_string[..]);

        let mut url = Url::parse(&base_url[..]).unwrap();
        url.set_query_from_pairs(query_pairs.iter().map(|(k, v)| (*k, *v)));

        let mut response = try!(http::get(format!("{}", url)));
        let mut response_body = String::new();
        response.read_to_string(&mut response_body).unwrap();

        match response.status {
            StatusCode::Ok => Ok(json::decode(&response_body).unwrap()),
            _ => Err(Error::Etcd(json::decode(&response_body).unwrap())),
        }
    }

    pub fn set(&self, key: &str, value: &str, ttl: Option<u64>) -> Result<Response, Error> {
        self.create_or_set(key, value, ttl, true)
    }

    // private

    fn build_url(&self, path: &str) -> String {
        format!("{}v2/keys{}", self.root_url, path)
    }

    fn create_or_set(
        &self,
        key: &str,
        value: &str,
        ttl: Option<u64>,
        prev_exist: bool,
    ) -> Result<Response, Error> {
        let url = self.build_url(key);
        let mut options = vec![];

        options.push(("value".to_string(), value.to_string()));

        if ttl.is_some() {
            options.push(("ttl".to_string(), format!("{}", ttl.unwrap())));
        }

        if !prev_exist {
            options.push(("prevExist".to_string(), "false".to_string()));
        }

        let body = serialize_owned(&options);

        let mut response = try!(http::put(url, body));
        let mut response_body = String::new();
        response.read_to_string(&mut response_body).unwrap();

        match response.status {
            StatusCode::Created | StatusCode::Ok => Ok(json::decode(&response_body).unwrap()),
            _ => Err(Error::Etcd(json::decode(&response_body).unwrap())),
        }
    }
}
<commit_msg>Make prev_exist an Option<bool>.<commit_after>use std::collections::HashMap;
use std::io::Read;

use hyper::status::StatusCode;
use rustc_serialize::json;
use url::{ParseError, Url};
use url::form_urlencoded::serialize_owned;

use error::Error;
use http;
use response::Response;

#[derive(Debug)]
pub struct Client {
    root_url: String,
}

impl Client {
    pub fn new(root_url: &str) -> Result<Client, ParseError> {
        let url = try!(Url::parse(root_url));
        let client = Client {
            root_url: format!("{}", url.serialize()),
        };

        Ok(client)
    }

    pub fn default() -> Client {
        Client {
            root_url: "http://127.0.0.1:2379/".to_string(),
        }
    }

    pub fn create(&self, key: &str, value: &str, ttl: Option<u64>) -> Result<Response, Error> {
        self.create_or_set(key, value, ttl, Some(false))
    }

    pub fn delete(&self, key: &str, recursive: bool) -> Result<Response, Error> {
        let url = self.build_url(key);
        let mut options = vec![];

        options.push(("recursive".to_string(), format!("{}", recursive)));

        let body = serialize_owned(&options);

        let mut response = try!(http::delete(url, body));
        let mut response_body = String::new();
        response.read_to_string(&mut response_body).unwrap();

        match response.status {
            StatusCode::Ok => Ok(json::decode(&response_body).unwrap()),
            _ => Err(Error::Etcd(json::decode(&response_body).unwrap())),
        }
    }

    pub fn get(&self, key: &str, sort: bool, recursive: bool) -> Result<Response, Error> {
        let base_url = self.build_url(key);
        let sort_string = format!("{}", sort);
        let recursive_string = format!("{}", recursive);

        let mut query_pairs = HashMap::new();

        query_pairs.insert("sorted", &sort_string[..]);
        query_pairs.insert("recursive", &recursive_string[..]);

        let mut url = Url::parse(&base_url[..]).unwrap();
        url.set_query_from_pairs(query_pairs.iter().map(|(k, v)| (*k, *v)));

        let mut response = try!(http::get(format!("{}", url)));
        let mut response_body = String::new();
        response.read_to_string(&mut response_body).unwrap();

        match response.status {
            StatusCode::Ok => Ok(json::decode(&response_body).unwrap()),
            _ => Err(Error::Etcd(json::decode(&response_body).unwrap())),
        }
    }

    pub fn set(&self, key: &str, value: &str, ttl: Option<u64>) -> Result<Response, Error> {
        self.create_or_set(key, value, ttl, None)
    }

    // private

    fn build_url(&self, path: &str) -> String {
        format!("{}v2/keys{}", self.root_url, path)
    }

    fn create_or_set(
        &self,
        key: &str,
        value: &str,
        ttl: Option<u64>,
        prev_exist: Option<bool>,
    ) -> Result<Response, Error> {
        let url = self.build_url(key);
        let mut options = vec![];

        options.push(("value".to_string(), value.to_string()));

        if ttl.is_some() {
            options.push(("ttl".to_string(), format!("{}", ttl.unwrap())));
        }

        if prev_exist.is_some() {
            options.push(("prevExist".to_string(), format!("{}", prev_exist.unwrap())));
        }

        let body = serialize_owned(&options);

        let mut response = try!(http::put(url, body));
        let mut response_body = String::new();
        response.read_to_string(&mut response_body).unwrap();

        match response.status {
            StatusCode::Created | StatusCode::Ok => Ok(json::decode(&response_body).unwrap()),
            _ => Err(Error::Etcd(json::decode(&response_body).unwrap())),
        }
    }
}
<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    cluster::Cluster,
    experiments::{Context, Experiment, ExperimentParam},
    instance,
    instance::Instance,
    stats::PrometheusRangeView,
    tx_emitter::{EmitJobRequest, TxStats},
    util::human_readable_bytes_per_sec,
};
use anyhow::{anyhow, Result};
use async_trait::async_trait;
use futures::{future::try_join_all, join};
use libra_logger::{info, warn};
use libra_time::duration_since_epoch;
use libra_trace::{
    trace::{find_peer_with_stage, random_node, trace_node},
    LibraTraceClient,
};
use rand::{rngs::ThreadRng, seq::SliceRandom};
use serde_json::Value;
use std::{
    collections::HashSet,
    fmt::{Display, Error, Formatter},
    time::Duration,
};
use structopt::StructOpt;
use tokio::task::JoinHandle;

#[derive(StructOpt, Debug)]
pub struct PerformanceBenchmarkParams {
    #[structopt(
        long,
        default_value = "0",
        help = "Percent of nodes which should be down"
    )]
    pub percent_nodes_down: usize,
    #[structopt(long, help = "Whether benchmark should perform trace")]
    pub trace: bool,
    #[structopt(long, help = "Whether benchmark should trace only one libra node")]
    pub trace_single: bool,
    #[structopt(
        long,
        help = "Whether benchmark should perform trace from elastic search logs"
    )]
    pub use_logs_for_trace: bool,
    #[structopt(
    long,
    default_value = Box::leak(format!("{}", DEFAULT_BENCH_DURATION).into_boxed_str()),
    help = "Duration of an experiment in seconds"
    )]
    pub duration: u64,
    #[structopt(long, help = "Set fixed tps during perf experiment")]
    pub tps: Option<u64>,
    #[structopt(
        long,
        help = "Whether benchmark should pick one node to run DB backup."
    )]
    pub backup: bool,
    #[structopt(long, default_value = "0", help = "Set gas price in tx")]
    pub gas_price: u64,
}

pub struct PerformanceBenchmark {
    down_validators: Vec<Instance>,
    up_validators: Vec<Instance>,
    up_fullnodes: Vec<Instance>,
    percent_nodes_down: usize,
    duration: Duration,
    trace: bool,
    trace_single: bool,
    tps: Option<u64>,
    use_logs_for_trace: bool,
    backup: bool,
    gas_price: u64,
}

pub const DEFAULT_BENCH_DURATION: u64 = 120;

impl PerformanceBenchmarkParams {
    pub fn new_nodes_down(percent_nodes_down: usize) -> Self {
        Self {
            percent_nodes_down,
            duration: DEFAULT_BENCH_DURATION,
            trace: false,
            trace_single: false,
            tps: None,
            use_logs_for_trace: false,
            backup: false,
            gas_price: 0,
        }
    }

    pub fn new_fixed_tps(percent_nodes_down: usize, fixed_tps: u64) -> Self {
        Self {
            percent_nodes_down,
            duration: DEFAULT_BENCH_DURATION,
            trace: false,
            trace_single: false,
            tps: Some(fixed_tps),
            use_logs_for_trace: false,
            backup: false,
            gas_price: 0,
        }
    }

    pub fn non_zero_gas_price(percent_nodes_down: usize, gas_price: u64) -> Self {
        Self {
            percent_nodes_down,
            duration: DEFAULT_BENCH_DURATION,
            trace: false,
            trace_single: false,
            tps: None,
            use_logs_for_trace: false,
            backup: false,
            gas_price,
        }
    }

    pub fn enable_db_backup(mut self) -> Self {
        self.backup = true;
        self
    }
}

impl ExperimentParam for PerformanceBenchmarkParams {
    type E = PerformanceBenchmark;
    fn build(self, cluster: &Cluster) -> Self::E {
        let all_fullnode_instances = cluster.fullnode_instances();
        let num_nodes = cluster.validator_instances().len();
        let nodes_down = (num_nodes * self.percent_nodes_down) / 100;
        let (down, up) = cluster.split_n_validators_random(nodes_down);
        let up_validators = up.into_validator_instances();
        let up_fullnodes: Vec<_> = up_validators
            .iter()
            .filter_map(|val| {
                all_fullnode_instances
                    .iter()
                    .find(|x| val.validator_group() == x.validator_group())
                    .cloned()
            })
            .collect();
        Self::E {
            down_validators: down.into_validator_instances(),
            up_validators,
            up_fullnodes,
            percent_nodes_down: self.percent_nodes_down,
            duration: Duration::from_secs(self.duration),
            trace: self.trace,
            trace_single: self.trace_single,
            tps: self.tps,
            use_logs_for_trace: self.use_logs_for_trace,
            backup: self.backup,
            gas_price: self.gas_price,
        }
    }
}

#[async_trait]
impl Experiment for PerformanceBenchmark {
    fn affected_validators(&self) -> HashSet<String> {
        instance::instancelist_to_set(&self.down_validators)
    }

    async fn run(&mut self, context: &mut Context<'_>) -> Result<()> {
        let futures: Vec<_> = self.down_validators.iter().map(Instance::stop).collect();
        try_join_all(futures).await?;

        let backup = self.maybe_start_backup()?;
        let buffer = Duration::from_secs(60);
        let window = self.duration + buffer * 2;
        let instances = if context.emit_to_validator {
            self.up_validators.clone()
        } else {
            self.up_fullnodes.clone()
        };
        let emit_job_request = match self.tps {
            Some(tps) => EmitJobRequest::fixed_tps(instances, tps, self.gas_price),
            None => EmitJobRequest::for_instances(
                instances,
                context.global_emit_job_request,
                self.gas_price,
            ),
        };
        let emit_txn = context.tx_emitter.emit_txn_for(window, emit_job_request);
        let start = chrono::Utc::now();
        let trace_tail = &context.trace_tail;
        let trace_delay = buffer;
        let trace = self.trace;
        let capture_trace = async move {
            if trace {
                tokio::time::delay_for(trace_delay).await;
                Some(trace_tail.capture_trace(Duration::from_secs(5)).await)
            } else {
                None
            }
        };
        let (stats, mut trace) = join!(emit_txn, capture_trace);

        // Trace
        let trace_log = self.use_logs_for_trace;
        if trace_log {
            let start = start + chrono::Duration::seconds(60);
            let libra_trace_client = LibraTraceClient::new("elasticsearch-master", 9200);
            trace = match libra_trace_client
                .get_libra_trace(start, chrono::Duration::seconds(5))
                .await
            {
                Ok(trace) => Some(trace),
                Err(err) => {
                    info!("Failed to capture traces from elastic search {}", err);
                    None
                }
            };
        }
        if let Some(trace) = trace {
            info!("Traced {} events", trace.len());
            let mut events = vec![];
            for (node, mut event) in trace {
                // This could be done more elegantly, but for now this will do
                event
                    .json
                    .as_object_mut()
                    .unwrap()
                    .insert("peer".to_string(), Value::String(node));
                events.push(event);
            }
            events.sort_by_key(|k| k.timestamp);
            let node =
                random_node(&events[..], "json-rpc::submit", "txn::").expect("No trace node found");
            info!("Tracing {}", node);
            if self.trace_single {
                let filter_peer =
                    find_peer_with_stage(&events[..], &node, "libra_vm::execute_block_impl")
                        .expect("Can not find peer with libra_vm::execute_block_impl")
                        .to_string();
                events = events
                    .into_iter()
                    .filter(|node| {
                        node.json
                            .as_object()
                            .unwrap()
                            .get("peer")
                            .unwrap()
                            .as_str()
                            .unwrap()
                            == filter_peer
                    })
                    .collect();
            }
            trace_node(&events[..], &node);
        }

        // Report
        self.report(context, buffer, window, stats?).await?;

        // Clean up
        drop(backup);
        let futures: Vec<_> = self.down_validators.iter().map(|ic| ic.start()).collect();
        try_join_all(futures).await?;

        Ok(())
    }

    fn deadline(&self) -> Duration {
        Duration::from_secs(600) + self.duration
    }
}

impl PerformanceBenchmark {
    fn maybe_start_backup(&self) -> Result<Option<JoinHandle<()>>> {
        if !self.backup {
            return Ok(None);
        }

        let mut rng = ThreadRng::default();
        let validator = self
            .up_validators
            .choose(&mut rng)
            .ok_or_else(|| anyhow!("No up validator."))?
            .clone();

        const COMMAND: &str = "/opt/libra/bin/db-backup coordinator run \
            --transaction-batch-size 20000 \
            --state-snapshot-interval 1 \
            local-fs --dir $(mktemp -d -t libra_backup_XXXXXXXX);";

        Ok(Some(tokio::spawn(async move {
            validator.exec(COMMAND, true).await.unwrap_or_else(|e| {
                let err_msg = e.to_string();
                if err_msg.ends_with("exit code Some(137)") {
                    info!("db-backup killed.");
                } else {
                    warn!("db-backup failed: {}", err_msg);
                }
            })
        })))
    }

    async fn report(
        &mut self,
        context: &mut Context<'_>,
        buffer: Duration,
        window: Duration,
        stats: TxStats,
    ) -> Result<()> {
        let end = duration_since_epoch() - buffer;
        let start = end - window + 2 * buffer;
        info!(
            "Link to dashboard : {}",
            context.prometheus.link_to_dashboard(start, end)
        );

        let pv = PrometheusRangeView::new(&context.prometheus, start, end);

        // Transaction stats
        if let Some(avg_txns_per_block) = pv.avg_txns_per_block() {
            context
                .report
                .report_metric(&self, "avg_txns_per_block", avg_txns_per_block);
        }
        let additional = if self.backup {
            // Backup throughput
            let bytes_per_sec = pv.avg_backup_bytes_per_second().unwrap_or(-1.0);
            context
                .report
                .report_metric(&self, "avg_backup_bytes_per_second", bytes_per_sec);
            format!(" backup: {},", human_readable_bytes_per_sec(bytes_per_sec))
        } else {
            "".to_string()
        };
        context
            .report
            .report_txn_stats(self.to_string(), stats, window, &additional);
        Ok(())
    }
}

impl Display for PerformanceBenchmark {
    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> {
        if let Some(tps) = self.tps {
            write!(f, "fixed tps {}", tps)?;
        } else if self.percent_nodes_down == 0 {
            write!(f, "all up")?;
        } else {
            write!(f, "{}% down", self.percent_nodes_down)?;
        }
        if self.gas_price != 0 {
            write!(f, ", gas price {}", self.gas_price)?;
        }
        Ok(())
    }
}
<commit_msg>[cluster-test] increase test duration for fullnode recovery<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    cluster::Cluster,
    experiments::{Context, Experiment, ExperimentParam},
    instance,
    instance::Instance,
    stats::PrometheusRangeView,
    tx_emitter::{EmitJobRequest, TxStats},
    util::human_readable_bytes_per_sec,
};
use anyhow::{anyhow, Result};
use async_trait::async_trait;
use futures::{future::try_join_all, join};
use libra_logger::{info, warn};
use libra_time::duration_since_epoch;
use libra_trace::{
    trace::{find_peer_with_stage, random_node, trace_node},
    LibraTraceClient,
};
use rand::{rngs::ThreadRng, seq::SliceRandom};
use serde_json::Value;
use std::{
    collections::HashSet,
    fmt::{Display, Error, Formatter},
    time::Duration,
};
use structopt::StructOpt;
use tokio::task::JoinHandle;

#[derive(StructOpt, Debug)]
pub struct PerformanceBenchmarkParams {
    #[structopt(
        long,
        default_value = "0",
        help = "Percent of nodes which should be down"
    )]
    pub percent_nodes_down: usize,
    #[structopt(long, help = "Whether benchmark should perform trace")]
    pub trace: bool,
    #[structopt(long, help = "Whether benchmark should trace only one libra node")]
    pub trace_single: bool,
    #[structopt(
        long,
        help = "Whether benchmark should perform trace from elastic search logs"
    )]
    pub use_logs_for_trace: bool,
    #[structopt(
    long,
    default_value = Box::leak(format!("{}", DEFAULT_BENCH_DURATION).into_boxed_str()),
    help = "Duration of an experiment in seconds"
    )]
    pub duration: u64,
    #[structopt(long, help = "Set fixed tps during perf experiment")]
    pub tps: Option<u64>,
    #[structopt(
        long,
        help = "Whether benchmark should pick one node to run DB backup."
    )]
    pub backup: bool,
    #[structopt(long, default_value = "0", help = "Set gas price in tx")]
    pub gas_price: u64,
}

pub struct PerformanceBenchmark {
    down_validators: Vec<Instance>,
    up_validators: Vec<Instance>,
    up_fullnodes: Vec<Instance>,
    percent_nodes_down: usize,
    duration: Duration,
    trace: bool,
    trace_single: bool,
    tps: Option<u64>,
    use_logs_for_trace: bool,
    backup: bool,
    gas_price: u64,
}

pub const DEFAULT_BENCH_DURATION: u64 = 120;

impl PerformanceBenchmarkParams {
    pub fn new_nodes_down(percent_nodes_down: usize) -> Self {
        Self {
            percent_nodes_down,
            duration: DEFAULT_BENCH_DURATION,
            trace: false,
            trace_single: false,
            tps: None,
            use_logs_for_trace: false,
            backup: false,
            gas_price: 0,
        }
    }

    pub fn new_fixed_tps(percent_nodes_down: usize, fixed_tps: u64) -> Self {
        Self {
            percent_nodes_down,
            duration: DEFAULT_BENCH_DURATION,
            trace: false,
            trace_single: false,
            tps: Some(fixed_tps),
            use_logs_for_trace: false,
            backup: false,
            gas_price: 0,
        }
    }

    pub fn non_zero_gas_price(percent_nodes_down: usize, gas_price: u64) -> Self {
        Self {
            percent_nodes_down,
            duration: DEFAULT_BENCH_DURATION,
            trace: false,
            trace_single: false,
            tps: None,
            use_logs_for_trace: false,
            backup: false,
            gas_price,
        }
    }

    pub fn enable_db_backup(mut self) -> Self {
        self.backup = true;
        self
    }
}

impl ExperimentParam for PerformanceBenchmarkParams {
    type E = PerformanceBenchmark;
    fn build(self, cluster: &Cluster) -> Self::E {
        let all_fullnode_instances = cluster.fullnode_instances();
        let num_nodes = cluster.validator_instances().len();
        let nodes_down = (num_nodes * self.percent_nodes_down) / 100;
        let (down, up) = cluster.split_n_validators_random(nodes_down);
        let up_validators = up.into_validator_instances();
        let up_fullnodes: Vec<_> = up_validators
            .iter()
            .filter_map(|val| {
                all_fullnode_instances
                    .iter()
                    .find(|x| val.validator_group() == x.validator_group())
                    .cloned()
            })
            .collect();
        Self::E {
            down_validators: down.into_validator_instances(),
            up_validators,
            up_fullnodes,
            percent_nodes_down: self.percent_nodes_down,
            duration: Duration::from_secs(self.duration),
            trace: self.trace,
            trace_single: self.trace_single,
            tps: self.tps,
            use_logs_for_trace: self.use_logs_for_trace,
            backup: self.backup,
            gas_price: self.gas_price,
        }
    }
}

#[async_trait]
impl Experiment for PerformanceBenchmark {
    fn affected_validators(&self) -> HashSet<String> {
        instance::instancelist_to_set(&self.down_validators)
    }

    async fn run(&mut self, context: &mut Context<'_>) -> Result<()> {
        let futures: Vec<_> = self.down_validators.iter().map(Instance::stop).collect();
        try_join_all(futures).await?;

        let backup = self.maybe_start_backup()?;
        let buffer = Duration::from_secs(60);
        let window = self.duration + buffer * 2;
        let instances = if context.emit_to_validator {
            self.up_validators.clone()
        } else {
            self.up_fullnodes.clone()
        };
        let emit_job_request = match self.tps {
            Some(tps) => EmitJobRequest::fixed_tps(instances, tps, self.gas_price),
            None => EmitJobRequest::for_instances(
                instances,
                context.global_emit_job_request,
                self.gas_price,
            ),
        };
        let emit_txn = context.tx_emitter.emit_txn_for(window, emit_job_request);
        let start = chrono::Utc::now();
        let trace_tail = &context.trace_tail;
        let trace_delay = buffer;
        let trace = self.trace;
        let capture_trace = async move {
            if trace {
                tokio::time::delay_for(trace_delay).await;
                Some(trace_tail.capture_trace(Duration::from_secs(5)).await)
            } else {
                None
            }
        };
        let (stats, mut trace) = join!(emit_txn, capture_trace);

        // Trace
        let trace_log = self.use_logs_for_trace;
        if trace_log {
            let start = start + chrono::Duration::seconds(60);
            let libra_trace_client = LibraTraceClient::new("elasticsearch-master", 9200);
            trace = match libra_trace_client
                .get_libra_trace(start, chrono::Duration::seconds(5))
                .await
            {
                Ok(trace) => Some(trace),
                Err(err) => {
                    info!("Failed to capture traces from elastic search {}", err);
                    None
                }
            };
        }
        if let Some(trace) = trace {
            info!("Traced {} events", trace.len());
            let mut events = vec![];
            for (node, mut event) in trace {
                // This could be done more elegantly, but for now this will do
                event
                    .json
                    .as_object_mut()
                    .unwrap()
                    .insert("peer".to_string(), Value::String(node));
                events.push(event);
            }
            events.sort_by_key(|k| k.timestamp);
            let node =
                random_node(&events[..], "json-rpc::submit", "txn::").expect("No trace node found");
            info!("Tracing {}", node);
            if self.trace_single {
                let filter_peer =
                    find_peer_with_stage(&events[..], &node, "libra_vm::execute_block_impl")
                        .expect("Can not find peer with libra_vm::execute_block_impl")
                        .to_string();
                events = events
                    .into_iter()
                    .filter(|node| {
                        node.json
                            .as_object()
                            .unwrap()
                            .get("peer")
                            .unwrap()
                            .as_str()
                            .unwrap()
                            == filter_peer
                    })
                    .collect();
            }
            trace_node(&events[..], &node);
        }

        // Report
        self.report(context, buffer, window, stats?).await?;

        // Clean up
        drop(backup);
        let futures: Vec<_> = self.down_validators.iter().map(|ic| ic.start()).collect();
        try_join_all(futures).await?;

        Ok(())
    }

    fn deadline(&self) -> Duration {
        Duration::from_secs(900) + self.duration
    }
}

impl PerformanceBenchmark {
    fn maybe_start_backup(&self) -> Result<Option<JoinHandle<()>>> {
        if !self.backup {
            return Ok(None);
        }

        let mut rng = ThreadRng::default();
        let validator = self
            .up_validators
            .choose(&mut rng)
            .ok_or_else(|| anyhow!("No up validator."))?
            .clone();

        const COMMAND: &str = "/opt/libra/bin/db-backup coordinator run \
            --transaction-batch-size 20000 \
            --state-snapshot-interval 1 \
            local-fs --dir $(mktemp -d -t libra_backup_XXXXXXXX);";

        Ok(Some(tokio::spawn(async move {
            validator.exec(COMMAND, true).await.unwrap_or_else(|e| {
                let err_msg = e.to_string();
                if err_msg.ends_with("exit code Some(137)") {
                    info!("db-backup killed.");
                } else {
                    warn!("db-backup failed: {}", err_msg);
                }
            })
        })))
    }

    async fn report(
        &mut self,
        context: &mut Context<'_>,
        buffer: Duration,
        window: Duration,
        stats: TxStats,
    ) -> Result<()> {
        let end = duration_since_epoch() - buffer;
        let start = end - window + 2 * buffer;
        info!(
            "Link to dashboard : {}",
            context.prometheus.link_to_dashboard(start, end)
        );

        let pv = PrometheusRangeView::new(&context.prometheus, start, end);

        // Transaction stats
        if let Some(avg_txns_per_block) = pv.avg_txns_per_block() {
            context
                .report
                .report_metric(&self, "avg_txns_per_block", avg_txns_per_block);
        }
        let additional = if self.backup {
            // Backup throughput
            let bytes_per_sec = pv.avg_backup_bytes_per_second().unwrap_or(-1.0);
            context
                .report
                .report_metric(&self, "avg_backup_bytes_per_second", bytes_per_sec);
            format!(" backup: {},", human_readable_bytes_per_sec(bytes_per_sec))
        } else {
            "".to_string()
        };
        context
            .report
            .report_txn_stats(self.to_string(), stats, window, &additional);
        Ok(())
    }
}

impl Display for PerformanceBenchmark {
    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> {
        if let Some(tps) = self.tps {
            write!(f, "fixed tps {}", tps)?;
        } else if self.percent_nodes_down == 0 {
            write!(f, "all up")?;
        } else {
            write!(f, "{}% down", self.percent_nodes_down)?;
        }
        if self.gas_price != 0 {
            write!(f, ", gas price {}", self.gas_price)?;
        }
        Ok(())
    }
}
<|endoftext|>