<commit_before>#include "nanocv.h"
#include "models/forward_network.h"
#include <boost/program_options.hpp>
#include <set>

using namespace ncv;

static void test_grad_params(
        const string_t& header, const string_t& loss_id, const model_t& model, accumulator_t& acc_params)
{
        random_t<size_t> rand(2, 16);

        const size_t n_tests = 64;
        const size_t n_samples = rand();

        const rloss_t rloss = loss_manager_t::instance().get(loss_id);
        const loss_t& loss = *rloss;

        const size_t psize = model.psize();
        const size_t isize = model.isize();
        const size_t osize = model.osize();

        vector_t params(psize);
        vectors_t targets(n_samples, vector_t(osize));
        tensors_t inputs(n_samples, tensor_t(model.idims(), model.irows(), model.icols()));

        // optimization problem (wrt parameters & inputs): size
        auto fn_params_size = [&] ()
        {
                return psize;
        };

        // optimization problem (wrt parameters & inputs): function value
        auto fn_params_fval = [&] (const vector_t& x)
        {
                acc_params.reset(x);
                acc_params.update(inputs, targets, loss);

                return acc_params.value();
        };

        // optimization problem (wrt parameters & inputs): function value & gradient
        auto fn_params_grad = [&] (const vector_t& x, vector_t& gx)
        {
                acc_params.reset(x);
                acc_params.update(inputs, targets, loss);

                gx = acc_params.vgrad();
                return acc_params.value();
        };

        // construct optimization problem: analytic gradient and finite difference approximation
        const opt_problem_t problem_analytic_params(fn_params_size, fn_params_fval, fn_params_grad);
        const opt_problem_t problem_aproxdif_params(fn_params_size, fn_params_fval);

        for (size_t t = 0; t < n_tests; t ++)
        {
                random_t<scalar_t> prgen(-1.0, +1.0);
                random_t<scalar_t> irgen(-0.1, +0.1);
                random_t<size_t> trgen(0, osize - 1);

                prgen(params.data(), params.data() + psize);
                for (vector_t& target : targets)
                {
                        target = ncv::class_target(trgen(), osize);
                }
                for (tensor_t& input : inputs)
                {
                        irgen(input.data(), input.data() + isize);
                }

                vector_t analytic_params_grad, aproxdif_params_grad;

                problem_analytic_params(params, analytic_params_grad);
                problem_aproxdif_params(params, aproxdif_params_grad);

                const scalar_t dg_params = (analytic_params_grad - aproxdif_params_grad).lpNorm<Eigen::Infinity>();
                const bool ok = math::almost_equal(dg_params, scalar_t(0));

                log_info() << header << " [" << (t + 1) << "/" << n_tests
                           << "]: samples = " << n_samples
                           << ", dg_params = " << dg_params << " (" << (ok ? "ERROR" : "OK") << ").";
        }
}

static void test_grad_inputs(const string_t& header, const string_t& loss_id, const model_t& model)
{
        rmodel_t rmodel_inputs = model.clone();

        const size_t n_tests = 64;
        const size_t n_samples = rand();

        const rloss_t rloss = loss_manager_t::instance().get(loss_id);
        const loss_t& loss = *rloss;

        const size_t psize = model.psize();
        const size_t isize = model.isize();
        const size_t osize = model.osize();

        vector_t params(psize);
        vector_t target(osize);
        tensor_t input(model.idims(), model.irows(), model.icols());

        // optimization problem (wrt parameters & inputs): size
        auto fn_inputs_size = [&] ()
        {
                return isize;
        };

        // optimization problem (wrt parameters & inputs): function value
        auto fn_inputs_fval = [&] (const vector_t& x)
        {
                rmodel_inputs->load_params(params);

                const vector_t output = rmodel_inputs->output(x).vector();

                return loss.value(target, output);
        };

        // optimization problem (wrt parameters & inputs): function value & gradient
        auto fn_inputs_grad = [&] (const vector_t& x, vector_t& gx)
        {
                rmodel_inputs->load_params(params);

                const vector_t output = rmodel_inputs->output(x).vector();

                gx = rmodel_inputs->igrad(loss.vgrad(target, output)).vector();
                return loss.value(target, output);
        };

        // construct optimization problem: analytic gradient and finite difference approximation
        const opt_problem_t problem_analytic_inputs(fn_inputs_size, fn_inputs_fval, fn_inputs_grad);
        const opt_problem_t problem_aproxdif_inputs(fn_inputs_size, fn_inputs_fval);

        for (size_t t = 0; t < n_tests; t ++)
        {
                random_t<scalar_t> prgen(-1.0, +1.0);
                random_t<scalar_t> irgen(-0.1, +0.1);
                random_t<size_t> trgen(0, osize - 1);

                prgen(params.data(), params.data() + psize);
                target = ncv::class_target(trgen(), osize);
                irgen(input.data(), input.data() + isize);

                vector_t analytic_inputs_grad, aproxdif_inputs_grad;

                problem_analytic_inputs(input.vector(), analytic_inputs_grad);
                problem_aproxdif_inputs(input.vector(), aproxdif_inputs_grad);

                const scalar_t dg_inputs = (analytic_inputs_grad - aproxdif_inputs_grad).lpNorm<Eigen::Infinity>();
                const scalar_t eps = 1e-6;

                log_info() << header << " [" << (t + 1) << "/" << n_tests
                           << "]: samples = " << n_samples
                           << ", dg_inputs = " << dg_inputs << " (" << (dg_inputs > eps ? "ERROR" : "OK") << ").";
        }
}

static void test_grad(const string_t& header, const string_t& loss_id, const model_t& model)
{
        // check all criteria
        const strings_t criteria = criterion_manager_t::instance().ids();
        for (const string_t& criterion : criteria)
        {
                random_t<size_t> rand(2, 16);
                const size_t n_threads = 1 + (rand() % 2);

                accumulator_t acc_params(model, n_threads, criterion, criterion_t::type::vgrad, 1.0);
                test_grad_params(header + "[criterion = " + criterion + "]", loss_id, model, acc_params);
        }

        // check gradients wrt the input
        test_grad_inputs(header, loss_id, model);
}

int main(int argc, char *argv[])
{
        ncv::init();

        const strings_t conv_layer_ids { "", "conv" };
        const strings_t pool_layer_ids { "", "pool-max", "pool-min", "pool-avg" };
        const strings_t full_layer_ids { "", "linear" };
        const strings_t actv_layer_ids { "", "act-unit", "act-tanh", "act-snorm", "act-splus" };
        const strings_t loss_ids = loss_manager_t::instance().ids();

        const color_mode cmd_color = color_mode::luma;
        const size_t cmd_irows = 10;
        const size_t cmd_icols = 10;
        const size_t cmd_outputs = 4;
        const size_t cmd_max_layers = 2;

        // evaluate the analytical gradient vs. the finite difference approximation for various:
        //      * convolution layers
        //      * pooling layers
        //      * fully connected layers
        //      * activation layers
        std::set<string_t> descs;
        for (size_t n_layers = 0; n_layers <= cmd_max_layers; n_layers ++)
        {
                for (const string_t& actv_layer_id : actv_layer_ids)
                {
                        for (const string_t& pool_layer_id : pool_layer_ids)
                        {
                                for (const string_t& conv_layer_id : conv_layer_ids)
                                {
                                        for (const string_t& full_layer_id : full_layer_ids)
                                        {
                                                string_t desc;

                                                // convolution part
                                                for (size_t l = 0; l < n_layers && !conv_layer_id.empty(); l ++)
                                                {
                                                        random_t<size_t> rgen(2, 3);

                                                        string_t params;
                                                        params += "dims=" + text::to_string(rgen());
                                                        params += (rgen() % 2 == 0) ? ",rows=3,cols=3" : ",rows=4,cols=4";

                                                        desc += conv_layer_id + ":" + params + ";";
                                                        desc += pool_layer_id + ";";
                                                        desc += actv_layer_id + ";";
                                                }

                                                // fully-connected part
                                                for (size_t l = 0; l < n_layers && !full_layer_id.empty(); l ++)
                                                {
                                                        random_t<size_t> rgen(1, 8);

                                                        string_t params;
                                                        params += "dims=" + text::to_string(rgen());

                                                        desc += full_layer_id + ":" + params + ";";
                                                        desc += actv_layer_id + ";";
                                                }

                                                desc += "linear:dims=" + text::to_string(cmd_outputs) + ";";
                                                desc += "softmax:type=global;";

                                                descs.insert(desc);
                                        }
                                }
                        }
                }
        }

        for (const string_t& desc : descs)
        {
                // create network
                forward_network_t network(desc);
                network.resize(cmd_irows, cmd_icols, cmd_outputs, cmd_color, true);

                // test network
                for (const string_t& loss_id : loss_ids)
                {
                        test_grad("[loss = " + loss_id + "]", loss_id, network);
                }
        }

        // OK
        log_info() << done;
        return EXIT_SUCCESS;
}
<commit_msg>fix logging message<commit_after>#include "nanocv.h"
#include "models/forward_network.h"
#include <boost/program_options.hpp>
#include <set>

using namespace ncv;

static void test_grad_params(
        const string_t& header, const string_t& loss_id, const model_t& model, accumulator_t& acc_params)
{
        random_t<size_t> rand(2, 16);

        const size_t n_tests = 64;
        const size_t n_samples = rand();

        const rloss_t rloss = loss_manager_t::instance().get(loss_id);
        const loss_t& loss = *rloss;

        const size_t psize = model.psize();
        const size_t isize = model.isize();
        const size_t osize = model.osize();

        vector_t params(psize);
        vectors_t targets(n_samples, vector_t(osize));
        tensors_t inputs(n_samples, tensor_t(model.idims(), model.irows(), model.icols()));

        // optimization problem (wrt parameters & inputs): size
        auto fn_params_size = [&] ()
        {
                return psize;
        };

        // optimization problem (wrt parameters & inputs): function value
        auto fn_params_fval = [&] (const vector_t& x)
        {
                acc_params.reset(x);
                acc_params.update(inputs, targets, loss);

                return acc_params.value();
        };

        // optimization problem (wrt parameters & inputs): function value & gradient
        auto fn_params_grad = [&] (const vector_t& x, vector_t& gx)
        {
                acc_params.reset(x);
                acc_params.update(inputs, targets, loss);

                gx = acc_params.vgrad();
                return acc_params.value();
        };

        // construct optimization problem: analytic gradient and finite difference approximation
        const opt_problem_t problem_analytic_params(fn_params_size, fn_params_fval, fn_params_grad);
        const opt_problem_t problem_aproxdif_params(fn_params_size, fn_params_fval);

        for (size_t t = 0; t < n_tests; t ++)
        {
                random_t<scalar_t> prgen(-1.0, +1.0);
                random_t<scalar_t> irgen(-0.1, +0.1);
                random_t<size_t> trgen(0, osize - 1);

                prgen(params.data(), params.data() + psize);
                for (vector_t& target : targets)
                {
                        target = ncv::class_target(trgen(), osize);
                }
                for (tensor_t& input : inputs)
                {
                        irgen(input.data(), input.data() + isize);
                }

                vector_t analytic_params_grad, aproxdif_params_grad;

                problem_analytic_params(params, analytic_params_grad);
                problem_aproxdif_params(params, aproxdif_params_grad);

                const scalar_t dg_params = (analytic_params_grad - aproxdif_params_grad).lpNorm<Eigen::Infinity>();
                const bool ok = math::almost_equal(dg_params, scalar_t(0));

                log_info() << header << " [" << (t + 1) << "/" << n_tests
                           << "]: samples = " << n_samples
                           << ", dg_params = " << dg_params << " (" << (ok ? "OK" : "ERROR") << ").";
        }
}

static void test_grad_inputs(const string_t& header, const string_t& loss_id, const model_t& model)
{
        rmodel_t rmodel_inputs = model.clone();

        const size_t n_tests = 64;
        const size_t n_samples = rand();

        const rloss_t rloss = loss_manager_t::instance().get(loss_id);
        const loss_t& loss = *rloss;

        const size_t psize = model.psize();
        const size_t isize = model.isize();
        const size_t osize = model.osize();

        vector_t params(psize);
        vector_t target(osize);
        tensor_t input(model.idims(), model.irows(), model.icols());

        // optimization problem (wrt parameters & inputs): size
        auto fn_inputs_size = [&] ()
        {
                return isize;
        };

        // optimization problem (wrt parameters & inputs): function value
        auto fn_inputs_fval = [&] (const vector_t& x)
        {
                rmodel_inputs->load_params(params);

                const vector_t output = rmodel_inputs->output(x).vector();

                return loss.value(target, output);
        };

        // optimization problem (wrt parameters & inputs): function value & gradient
        auto fn_inputs_grad = [&] (const vector_t& x, vector_t& gx)
        {
                rmodel_inputs->load_params(params);

                const vector_t output = rmodel_inputs->output(x).vector();

                gx = rmodel_inputs->igrad(loss.vgrad(target, output)).vector();
                return loss.value(target, output);
        };

        // construct optimization problem: analytic gradient and finite difference approximation
        const opt_problem_t problem_analytic_inputs(fn_inputs_size, fn_inputs_fval, fn_inputs_grad);
        const opt_problem_t problem_aproxdif_inputs(fn_inputs_size, fn_inputs_fval);

        for (size_t t = 0; t < n_tests; t ++)
        {
                random_t<scalar_t> prgen(-1.0, +1.0);
                random_t<scalar_t> irgen(-0.1, +0.1);
                random_t<size_t> trgen(0, osize - 1);

                prgen(params.data(), params.data() + psize);
                target = ncv::class_target(trgen(), osize);
                irgen(input.data(), input.data() + isize);

                vector_t analytic_inputs_grad, aproxdif_inputs_grad;

                problem_analytic_inputs(input.vector(), analytic_inputs_grad);
                problem_aproxdif_inputs(input.vector(), aproxdif_inputs_grad);

                const scalar_t dg_inputs = (analytic_inputs_grad - aproxdif_inputs_grad).lpNorm<Eigen::Infinity>();
                const scalar_t eps = 1e-6;

                log_info() << header << " [" << (t + 1) << "/" << n_tests
                           << "]: samples = " << n_samples
                           << ", dg_inputs = " << dg_inputs << " (" << (dg_inputs > eps ? "ERROR" : "OK") << ").";
        }
}

static void test_grad(const string_t& header, const string_t& loss_id, const model_t& model)
{
        // check all criteria
        const strings_t criteria = criterion_manager_t::instance().ids();
        for (const string_t& criterion : criteria)
        {
                random_t<size_t> rand(2, 16);
                const size_t n_threads = 1 + (rand() % 2);

                accumulator_t acc_params(model, n_threads, criterion, criterion_t::type::vgrad, 1.0);
                test_grad_params(header + "[criterion = " + criterion + "]", loss_id, model, acc_params);
        }

        // check gradients wrt the input
        test_grad_inputs(header, loss_id, model);
}

int main(int argc, char *argv[])
{
        ncv::init();

        const strings_t conv_layer_ids { "", "conv" };
        const strings_t pool_layer_ids { "", "pool-max", "pool-min", "pool-avg" };
        const strings_t full_layer_ids { "", "linear" };
        const strings_t actv_layer_ids { "", "act-unit", "act-tanh", "act-snorm", "act-splus" };
        const strings_t loss_ids = loss_manager_t::instance().ids();

        const color_mode cmd_color = color_mode::luma;
        const size_t cmd_irows = 10;
        const size_t cmd_icols = 10;
        const size_t cmd_outputs = 4;
        const size_t cmd_max_layers = 2;

        // evaluate the analytical gradient vs. the finite difference approximation for various:
        //      * convolution layers
        //      * pooling layers
        //      * fully connected layers
        //      * activation layers
        std::set<string_t> descs;
        for (size_t n_layers = 0; n_layers <= cmd_max_layers; n_layers ++)
        {
                for (const string_t& actv_layer_id : actv_layer_ids)
                {
                        for (const string_t& pool_layer_id : pool_layer_ids)
                        {
                                for (const string_t& conv_layer_id : conv_layer_ids)
                                {
                                        for (const string_t& full_layer_id : full_layer_ids)
                                        {
                                                string_t desc;

                                                // convolution part
                                                for (size_t l = 0; l < n_layers && !conv_layer_id.empty(); l ++)
                                                {
                                                        random_t<size_t> rgen(2, 3);

                                                        string_t params;
                                                        params += "dims=" + text::to_string(rgen());
                                                        params += (rgen() % 2 == 0) ? ",rows=3,cols=3" : ",rows=4,cols=4";

                                                        desc += conv_layer_id + ":" + params + ";";
                                                        desc += pool_layer_id + ";";
                                                        desc += actv_layer_id + ";";
                                                }

                                                // fully-connected part
                                                for (size_t l = 0; l < n_layers && !full_layer_id.empty(); l ++)
                                                {
                                                        random_t<size_t> rgen(1, 8);

                                                        string_t params;
                                                        params += "dims=" + text::to_string(rgen());

                                                        desc += full_layer_id + ":" + params + ";";
                                                        desc += actv_layer_id + ";";
                                                }

                                                desc += "linear:dims=" + text::to_string(cmd_outputs) + ";";
                                                desc += "softmax:type=global;";

                                                descs.insert(desc);
                                        }
                                }
                        }
                }
        }

        for (const string_t& desc : descs)
        {
                // create network
                forward_network_t network(desc);
                network.resize(cmd_irows, cmd_icols, cmd_outputs, cmd_color, true);

                // test network
                for (const string_t& loss_id : loss_ids)
                {
                        test_grad("[loss = " + loss_id + "]", loss_id, network);
                }
        }

        // OK
        log_info() << done;
        return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <boost/range/irange.hpp>

#include <seastar/util/defer.hh>
#include <seastar/core/app-template.hh>
#include <seastar/core/thread.hh>

#include "partition_slice_builder.hh"
#include "schema_builder.hh"
#include "memtable.hh"
#include "row_cache.hh"
#include "frozen_mutation.hh"
#include "test/lib/tmpdir.hh"
#include "sstables/sstables.hh"
#include "canonical_mutation.hh"
#include "test/lib/sstable_utils.hh"
#include "test/lib/test_services.hh"
#include "test/lib/sstable_test_env.hh"

class size_calculator {
    class nest {
    public:
        static thread_local int level;
        nest() { ++level; }
        ~nest() { --level; }
    };

    static std::string prefix() {
        std::string s(" ");
        for (int i = 0; i < nest::level; ++i) {
            s += "-- ";
        }
        return s;
    }
public:
    static void print_cache_entry_size() {
        std::cout << prefix() << "sizeof(cache_entry) = " << sizeof(cache_entry) << "\n";

        {
            nest n;
            std::cout << prefix() << "sizeof(decorated_key) = " << sizeof(dht::decorated_key) << "\n";
            std::cout << prefix() << "sizeof(cache_link_type) = " << sizeof(cache_entry::cache_link_type) << "\n";
            print_mutation_partition_size();
        }

        std::cout << "\n";

        std::cout << prefix() << "sizeof(rows_entry) = " << sizeof(rows_entry) << "\n";
        std::cout << prefix() << "sizeof(lru_link_type) = " << sizeof(rows_entry::lru_link_type) << "\n";
        std::cout << prefix() << "sizeof(deletable_row) = " << sizeof(deletable_row) << "\n";
        std::cout << prefix() << "sizeof(row) = " << sizeof(row) << "\n";
        std::cout << prefix() << "sizeof(atomic_cell_or_collection) = " << sizeof(atomic_cell_or_collection) << "\n";
    }

    static void print_mutation_partition_size() {
        std::cout << prefix() << "sizeof(mutation_partition) = " << sizeof(mutation_partition) << "\n";
        {
            nest n;
            std::cout << prefix() << "sizeof(_static_row) = " << sizeof(mutation_partition::_static_row) << "\n";
            std::cout << prefix() << "sizeof(_rows) = " << sizeof(mutation_partition::_rows) << "\n";
            std::cout << prefix() << "sizeof(_row_tombstones) = " << sizeof(mutation_partition::_row_tombstones) <<
            "\n";
        }
    }
};

thread_local int size_calculator::nest::level = 0;

static schema_ptr cassandra_stress_schema() {
    return schema_builder("ks", "cf")
        .with_column("KEY", bytes_type, column_kind::partition_key)
        .with_column("C0", bytes_type)
        .with_column("C1", bytes_type)
        .with_column("C2", bytes_type)
        .with_column("C3", bytes_type)
        .with_column("C4", bytes_type)
        .build();
}

[[gnu::unused]]
static mutation make_cs_mutation() {
    auto s = cassandra_stress_schema();
    mutation m(s, partition_key::from_single_value(*s, bytes_type->from_string("4b343050393536353531")));
    for (auto&& col : s->regular_columns()) {
        m.set_clustered_cell(clustering_key::make_empty(), col,
            atomic_cell::make_live(*bytes_type, 1, bytes_type->from_string("8f75da6b3dcec90c8a404fb9a5f6b0621e62d39c69ba5758e5f41b78311fbb26cc7a")));
    }
    return m;
}

bytes random_bytes(size_t size) {
    bytes result(bytes::initialized_later(), size);
    for (size_t i = 0; i < size; ++i) {
        result[i] = std::rand() % std::numeric_limits<uint8_t>::max();
    }
    return result;
}

sstring random_string(size_t size) {
    sstring result = uninitialized_string(size);
    static const char chars[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
    for (size_t i = 0; i < size; ++i) {
        result[i] = chars[std::rand() % sizeof(chars)];
    }
    return result;
}

struct mutation_settings {
    size_t column_count;
    size_t column_name_size;
    size_t row_count;
    size_t partition_key_size;
    size_t clustering_key_size;
    size_t data_size;
};

static mutation make_mutation(mutation_settings settings) {
    auto builder = schema_builder("ks", "cf")
        .with_column("pk", bytes_type, column_kind::partition_key)
        .with_column("ck", bytes_type, column_kind::clustering_key);

    for (size_t i = 0; i < settings.column_count; ++i) {
        builder.with_column(to_bytes(random_string(settings.column_name_size)), bytes_type);
    }

    auto s = builder.build();

    mutation m(s, partition_key::from_single_value(*s, bytes_type->decompose(data_value(random_bytes(settings.partition_key_size)))));

    for (size_t i = 0; i < settings.row_count; ++i) {
        auto ck = clustering_key::from_single_value(*s, bytes_type->decompose(data_value(random_bytes(settings.clustering_key_size))));
        for (auto&& col : s->regular_columns()) {
            m.set_clustered_cell(ck, col,
                atomic_cell::make_live(*bytes_type, 1,
                    bytes_type->decompose(data_value(random_bytes(settings.data_size)))));
        }
    }
    return m;
}

struct sizes {
    size_t memtable;
    size_t cache;
    size_t sstable;
    size_t frozen;
    size_t canonical;
    size_t query_result;
};

static sizes calculate_sizes(const mutation& m) {
    sizes result;
    auto s = m.schema();
    auto mt = make_lw_shared<memtable>(s);
    cache_tracker tracker;
    row_cache cache(s, make_empty_snapshot_source(), tracker);

    auto cache_initial_occupancy = tracker.region().occupancy().used_space();

    assert(mt->occupancy().used_space() == 0);

    mt->apply(m);
    cache.populate(m);

    result.memtable = mt->occupancy().used_space();
    result.cache = tracker.region().occupancy().used_space() - cache_initial_occupancy;
    result.frozen = freeze(m).representation().size();
    result.canonical = canonical_mutation(m).representation().size();
    result.query_result = m.query(partition_slice_builder(*s).build(), query::result_options::only_result()).buf().size();

    tmpdir sstable_dir;
    sstables::test_env env;
    auto sst = env.make_sstable(s,
        sstable_dir.path().string(),
        1 /* generation */,
        sstables::sstable::version_types::la,
        sstables::sstable::format_types::big);
    write_memtable_to_sstable_for_test(*mt, sst).get();
    sst->load().get();
    result.sstable = sst->data_size();

    return result;
}

int main(int argc, char** argv) {
    namespace bpo = boost::program_options;
    app_template app;
    app.add_options()
        ("column-count", bpo::value<size_t>()->default_value(5), "column count")
        ("column-name-size", bpo::value<size_t>()->default_value(2), "column name size")
        ("row-count", bpo::value<size_t>()->default_value(1), "row count")
        ("partition-key-size", bpo::value<size_t>()->default_value(10), "partition key size")
        ("clustering-key-size", bpo::value<size_t>()->default_value(10), "clustering key size")
        ("data-size", bpo::value<size_t>()->default_value(32), "cell data size");

    return app.run(argc, argv, [&] {
        if (smp::count != 1) {
            throw std::runtime_error("This test has to be run with -c1");
        }
        return seastar::async([&] {
            storage_service_for_tests ssft;

            mutation_settings settings;
            settings.column_count = app.configuration()["column-count"].as<size_t>();
            settings.column_name_size = app.configuration()["column-name-size"].as<size_t>();
            settings.row_count = app.configuration()["row-count"].as<size_t>();
            settings.partition_key_size = app.configuration()["partition-key-size"].as<size_t>();
            settings.clustering_key_size = app.configuration()["clustering-key-size"].as<size_t>();
            settings.data_size = app.configuration()["data-size"].as<size_t>();

            auto m = make_mutation(settings);
            auto sizes = calculate_sizes(m);

            std::cout << "mutation footprint:" << "\n";
            std::cout << " - in cache:     " << sizes.cache << "\n";
            std::cout << " - in memtable:  " << sizes.memtable << "\n";
            std::cout << " - in sstable:   " << sizes.sstable << "\n";
            std::cout << " - frozen:       " << sizes.frozen << "\n";
            std::cout << " - canonical:    " << sizes.canonical << "\n";
            std::cout << " - query result: " << sizes.query_result << "\n";

            std::cout << "\n";
            size_calculator::print_cache_entry_size();
        });
    });
}
<commit_msg>test: memory_footprint: Calculate sstable size for each format version<commit_after>/*
 * Copyright (C) 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <boost/range/irange.hpp>

#include <seastar/util/defer.hh>
#include <seastar/core/app-template.hh>
#include <seastar/core/thread.hh>

#include "partition_slice_builder.hh"
#include "schema_builder.hh"
#include "memtable.hh"
#include "row_cache.hh"
#include "frozen_mutation.hh"
#include "test/lib/tmpdir.hh"
#include "sstables/sstables.hh"
#include "canonical_mutation.hh"
#include "test/lib/sstable_utils.hh"
#include "test/lib/test_services.hh"
#include "test/lib/sstable_test_env.hh"

class size_calculator {
    class nest {
    public:
        static thread_local int level;
        nest() { ++level; }
        ~nest() { --level; }
    };

    static std::string prefix() {
        std::string s(" ");
        for (int i = 0; i < nest::level; ++i) {
            s += "-- ";
        }
        return s;
    }
public:
    static void print_cache_entry_size() {
        std::cout << prefix() << "sizeof(cache_entry) = " << sizeof(cache_entry) << "\n";

        {
            nest n;
            std::cout << prefix() << "sizeof(decorated_key) = " << sizeof(dht::decorated_key) << "\n";
            std::cout << prefix() << "sizeof(cache_link_type) = " << sizeof(cache_entry::cache_link_type) << "\n";
            print_mutation_partition_size();
        }

        std::cout << "\n";

        std::cout << prefix() << "sizeof(rows_entry) = " << sizeof(rows_entry) << "\n";
        std::cout << prefix() << "sizeof(lru_link_type) = " << sizeof(rows_entry::lru_link_type) << "\n";
        std::cout << prefix() << "sizeof(deletable_row) = " << sizeof(deletable_row) << "\n";
        std::cout << prefix() << "sizeof(row) = " << sizeof(row) << "\n";
        std::cout << prefix() << "sizeof(atomic_cell_or_collection) = " << sizeof(atomic_cell_or_collection) << "\n";
    }

    static void print_mutation_partition_size() {
        std::cout << prefix() << "sizeof(mutation_partition) = " << sizeof(mutation_partition) << "\n";
        {
            nest n;
            std::cout << prefix() << "sizeof(_static_row) = " << sizeof(mutation_partition::_static_row) << "\n";
            std::cout << prefix() << "sizeof(_rows) = " << sizeof(mutation_partition::_rows) << "\n";
            std::cout << prefix() << "sizeof(_row_tombstones) = " << sizeof(mutation_partition::_row_tombstones) <<
            "\n";
        }
    }
};

thread_local int size_calculator::nest::level = 0;

static schema_ptr cassandra_stress_schema() {
    return schema_builder("ks", "cf")
        .with_column("KEY", bytes_type, column_kind::partition_key)
        .with_column("C0", bytes_type)
        .with_column("C1", bytes_type)
        .with_column("C2", bytes_type)
        .with_column("C3", bytes_type)
        .with_column("C4", bytes_type)
        .build();
}

[[gnu::unused]]
static mutation make_cs_mutation() {
    auto s = cassandra_stress_schema();
    mutation m(s, partition_key::from_single_value(*s, bytes_type->from_string("4b343050393536353531")));
    for (auto&& col : s->regular_columns()) {
        m.set_clustered_cell(clustering_key::make_empty(), col,
            atomic_cell::make_live(*bytes_type, 1, bytes_type->from_string("8f75da6b3dcec90c8a404fb9a5f6b0621e62d39c69ba5758e5f41b78311fbb26cc7a")));
    }
    return m;
}

bytes random_bytes(size_t size) {
    bytes result(bytes::initialized_later(), size);
    for (size_t i = 0; i < size; ++i) {
        result[i] = std::rand() % std::numeric_limits<uint8_t>::max();
    }
    return result;
}

sstring random_string(size_t size) {
    sstring result = uninitialized_string(size);
    static const char chars[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
    for (size_t i = 0; i < size; ++i) {
        result[i] = chars[std::rand() % sizeof(chars)];
    }
    return result;
}

struct mutation_settings {
    size_t column_count;
    size_t column_name_size;
    size_t row_count;
    size_t partition_key_size;
    size_t clustering_key_size;
    size_t data_size;
};

static mutation make_mutation(mutation_settings settings) {
    auto builder = schema_builder("ks", "cf")
        .with_column("pk", bytes_type, column_kind::partition_key)
        .with_column("ck", bytes_type, column_kind::clustering_key);

    for (size_t i = 0; i < settings.column_count; ++i) {
        builder.with_column(to_bytes(random_string(settings.column_name_size)), bytes_type);
    }

    auto s = builder.build();

    mutation m(s, partition_key::from_single_value(*s, bytes_type->decompose(data_value(random_bytes(settings.partition_key_size)))));

    for (size_t i = 0; i < settings.row_count; ++i) {
        auto ck = clustering_key::from_single_value(*s, bytes_type->decompose(data_value(random_bytes(settings.clustering_key_size))));
        for (auto&& col : s->regular_columns()) {
            m.set_clustered_cell(ck, col,
                atomic_cell::make_live(*bytes_type, 1,
                    bytes_type->decompose(data_value(random_bytes(settings.data_size)))));
        }
    }
    return m;
}

struct sizes {
    size_t memtable;
    size_t cache;
    std::map<sstables::sstable::version_types, size_t> sstable;
    size_t frozen;
    size_t canonical;
    size_t query_result;
};

static sizes calculate_sizes(const mutation& m) {
    sizes result;
    auto s = m.schema();
    auto mt = make_lw_shared<memtable>(s);
    cache_tracker tracker;
    row_cache cache(s, make_empty_snapshot_source(), tracker);

    auto cache_initial_occupancy = tracker.region().occupancy().used_space();

    assert(mt->occupancy().used_space() == 0);

    mt->apply(m);
    cache.populate(m);

    result.memtable = mt->occupancy().used_space();
    result.cache = tracker.region().occupancy().used_space() - cache_initial_occupancy;
    result.frozen = freeze(m).representation().size();
    result.canonical = canonical_mutation(m).representation().size();
    result.query_result = m.query(partition_slice_builder(*s).build(), query::result_options::only_result()).buf().size();

    tmpdir sstable_dir;
    sstables::test_env env;
    for (auto v  : sstables::all_sstable_versions) {
        auto sst = env.make_sstable(s,
            sstable_dir.path().string(),
            1 /* generation */,
            v,
            sstables::sstable::format_types::big);
        write_memtable_to_sstable_for_test(*mt, sst).get();
        sst->load().get();
        result.sstable[v] = sst->data_size();
    }

    return result;
}

int main(int argc, char** argv) {
    namespace bpo = boost::program_options;
    app_template app;
    app.add_options()
        ("column-count", bpo::value<size_t>()->default_value(5), "column count")
        ("column-name-size", bpo::value<size_t>()->default_value(2), "column name size")
        ("row-count", bpo::value<size_t>()->default_value(1), "row count")
        ("partition-key-size", bpo::value<size_t>()->default_value(10), "partition key size")
        ("clustering-key-size", bpo::value<size_t>()->default_value(10), "clustering key size")
        ("data-size", bpo::value<size_t>()->default_value(32), "cell data size");

    return app.run(argc, argv, [&] {
        if (smp::count != 1) {
            throw std::runtime_error("This test has to be run with -c1");
        }
        return seastar::async([&] {
            storage_service_for_tests ssft;

            mutation_settings settings;
            settings.column_count = app.configuration()["column-count"].as<size_t>();
            settings.column_name_size = app.configuration()["column-name-size"].as<size_t>();
            settings.row_count = app.configuration()["row-count"].as<size_t>();
            settings.partition_key_size = app.configuration()["partition-key-size"].as<size_t>();
            settings.clustering_key_size = app.configuration()["clustering-key-size"].as<size_t>();
            settings.data_size = app.configuration()["data-size"].as<size_t>();

            auto m = make_mutation(settings);
            auto sizes = calculate_sizes(m);

            std::cout << "mutation footprint:" << "\n";
            std::cout << " - in cache:     " << sizes.cache << "\n";
            std::cout << " - in memtable:  " << sizes.memtable << "\n";
            std::cout << " - in sstable:\n";
            for (auto v : sizes.sstable) {
                std::cout << "   " << sstables::to_string(v.first) << ":   " << v.second << "\n";
            }
            std::cout << " - frozen:       " << sizes.frozen << "\n";
            std::cout << " - canonical:    " << sizes.canonical << "\n";
            std::cout << " - query result: " << sizes.query_result << "\n";

            std::cout << "\n";
            size_calculator::print_cache_entry_size();
        });
    });
}
<|endoftext|>


<commit_before>/*
  TwoWire.cpp - TWI/I2C library for Arduino & Wiring
  Copyright (c) 2006 Nicholas Zambetti.  All right reserved.

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

  Modified 2012 by Todd Krein (todd@krein.org) to implement repeated starts
  Modified December 2014 by Ivan Grokhotkov (ivan@esp8266.com) - esp8266 support
  Modified April 2015 by Hrsto Gochkov (ficeto@ficeto.com) - alternative esp8266 support
*/

extern "C" {
  #include <stdlib.h>
  #include <string.h>
  #include <inttypes.h>
}

#include "twi.h"
#include "Wire.h"

// Initialize Class Variables //////////////////////////////////////////////////

uint8_t TwoWire::rxBuffer[BUFFER_LENGTH];
uint8_t TwoWire::rxBufferIndex = 0;
uint8_t TwoWire::rxBufferLength = 0;

uint8_t TwoWire::txAddress = 0;
uint8_t TwoWire::txBuffer[BUFFER_LENGTH];
uint8_t TwoWire::txBufferIndex = 0;
uint8_t TwoWire::txBufferLength = 0;

uint8_t TwoWire::transmitting = 0;
void (*TwoWire::user_onRequest)(void);
void (*TwoWire::user_onReceive)(int);

static int default_sda_pin = SDA;
static int default_scl_pin = SCL;

// Constructors ////////////////////////////////////////////////////////////////

TwoWire::TwoWire(){}

// Public Methods //////////////////////////////////////////////////////////////

void TwoWire::begin(int sda, int scl){
  twi_init(sda, scl);
  flush();
}

void TwoWire::pins(int sda, int scl){
  default_sda_pin = sda;
  default_scl_pin = scl;
}

void TwoWire::begin(void){
  begin(default_sda_pin, default_scl_pin);
}

void TwoWire::begin(uint8_t address){
  // twi_setAddress(address);
  // twi_attachSlaveTxEvent(onRequestService);
  // twi_attachSlaveRxEvent(onReceiveService);
  begin();
}

void TwoWire::begin(int address){
  begin((uint8_t)address);
}

void TwoWire::setClock(uint32_t frequency){
  twi_setClock(frequency);
}

size_t TwoWire::requestFrom(uint8_t address, size_t size, bool sendStop){
  if(size > BUFFER_LENGTH){
    size = BUFFER_LENGTH;
  }
  size_t read = (twi_readFrom(address, rxBuffer, size, sendStop) == 0)?size:0;
  rxBufferIndex = 0;
  rxBufferLength = read;
  return read;
}

uint8_t TwoWire::requestFrom(uint8_t address, uint8_t quantity, uint8_t sendStop){
  return requestFrom(address, static_cast<size_t>(quantity), static_cast<bool>(sendStop));
}

uint8_t TwoWire::requestFrom(uint8_t address, uint8_t quantity){
  return requestFrom(address, static_cast<size_t>(quantity), true);
}

uint8_t TwoWire::requestFrom(int address, int quantity){
  return requestFrom(static_cast<uint8_t>(address), static_cast<size_t>(quantity), true);
}

uint8_t TwoWire::requestFrom(int address, int quantity, int sendStop){
  return requestFrom(static_cast<uint8_t>(address), static_cast<size_t>(quantity), static_cast<bool>(sendStop));
}

void TwoWire::beginTransmission(uint8_t address){
  transmitting = 1;
  txAddress = address;
  txBufferIndex = 0;
  txBufferLength = 0;
}

void TwoWire::beginTransmission(int address){
  beginTransmission((uint8_t)address);
}

uint8_t TwoWire::endTransmission(uint8_t sendStop){
  int8_t ret = twi_writeTo(txAddress, txBuffer, txBufferLength, sendStop);
  txBufferIndex = 0;
  txBufferLength = 0;
  transmitting = 0;
  return ret;
}

uint8_t TwoWire::endTransmission(void){
  return endTransmission(true);
}

size_t TwoWire::write(uint8_t data){
  if(transmitting){
    if(txBufferLength >= BUFFER_LENGTH){
      setWriteError();
      return 0;
    }
    txBuffer[txBufferIndex] = data;
    ++txBufferIndex;
    txBufferLength = txBufferIndex;
  } else {
    // i2c_slave_transmit(&data, 1);
  }
  return 1;
}

size_t TwoWire::write(const uint8_t *data, size_t quantity){
  if(transmitting){
    for(size_t i = 0; i < quantity; ++i){
      if(!write(data[i])) return i;
    }
  }else{
    // i2c_slave_transmit(data, quantity);
  }
  return quantity;
}

int TwoWire::available(void){
  return rxBufferLength - rxBufferIndex;
}

int TwoWire::read(void){
  int value = -1;
  if(rxBufferIndex < rxBufferLength){
    value = rxBuffer[rxBufferIndex];
    ++rxBufferIndex;
  }
  return value;
}

int TwoWire::peek(void){
  int value = -1;
  if(rxBufferIndex < rxBufferLength){
    value = rxBuffer[rxBufferIndex];
  }
  return value;
}

void TwoWire::flush(void){
  rxBufferIndex = 0;
  rxBufferLength = 0;
  txBufferIndex = 0;
  txBufferLength = 0;
}

void TwoWire::onReceiveService(uint8_t* inBytes, int numBytes)
{
  // don't bother if user hasn't registered a callback
  // if(!user_onReceive){
  //   return;
  // }
  // // don't bother if rx buffer is in use by a master requestFrom() op
  // // i know this drops data, but it allows for slight stupidity
  // // meaning, they may not have read all the master requestFrom() data yet
  // if(rxBufferIndex < rxBufferLength){
  //   return;
  // }
  // // copy twi rx buffer into local read buffer
  // // this enables new reads to happen in parallel
  // for(uint8_t i = 0; i < numBytes; ++i){
  //   rxBuffer[i] = inBytes[i];
  // }
  // // set rx iterator vars
  // rxBufferIndex = 0;
  // rxBufferLength = numBytes;
  // // alert user program
  // user_onReceive(numBytes);
}

void TwoWire::onRequestService(void){
  // // don't bother if user hasn't registered a callback
  // if(!user_onRequest){
  //   return;
  // }
  // // reset tx buffer iterator vars
  // // !!! this will kill any pending pre-master sendTo() activity
  // txBufferIndex = 0;
  // txBufferLength = 0;
  // // alert user program
  // user_onRequest();
}

void TwoWire::onReceive( void (*function)(int) ){
  //user_onReceive = function;
}

void TwoWire::onRequest( void (*function)(void) ){
  //user_onRequest = function;
}

// Preinstantiate Objects //////////////////////////////////////////////////////

TwoWire Wire = TwoWire();

<commit_msg>fix I2C case where using different pins would not work with libs calling begin internally<commit_after>/*
  TwoWire.cpp - TWI/I2C library for Arduino & Wiring
  Copyright (c) 2006 Nicholas Zambetti.  All right reserved.

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

  Modified 2012 by Todd Krein (todd@krein.org) to implement repeated starts
  Modified December 2014 by Ivan Grokhotkov (ivan@esp8266.com) - esp8266 support
  Modified April 2015 by Hrsto Gochkov (ficeto@ficeto.com) - alternative esp8266 support
*/

extern "C" {
  #include <stdlib.h>
  #include <string.h>
  #include <inttypes.h>
}

#include "twi.h"
#include "Wire.h"

// Initialize Class Variables //////////////////////////////////////////////////

uint8_t TwoWire::rxBuffer[BUFFER_LENGTH];
uint8_t TwoWire::rxBufferIndex = 0;
uint8_t TwoWire::rxBufferLength = 0;

uint8_t TwoWire::txAddress = 0;
uint8_t TwoWire::txBuffer[BUFFER_LENGTH];
uint8_t TwoWire::txBufferIndex = 0;
uint8_t TwoWire::txBufferLength = 0;

uint8_t TwoWire::transmitting = 0;
void (*TwoWire::user_onRequest)(void);
void (*TwoWire::user_onReceive)(int);

static int default_sda_pin = SDA;
static int default_scl_pin = SCL;

// Constructors ////////////////////////////////////////////////////////////////

TwoWire::TwoWire(){}

// Public Methods //////////////////////////////////////////////////////////////

void TwoWire::begin(int sda, int scl){
  default_sda_pin = sda;
  default_scl_pin = scl;
  twi_init(sda, scl);
  flush();
}

void TwoWire::pins(int sda, int scl){
  default_sda_pin = sda;
  default_scl_pin = scl;
}

void TwoWire::begin(void){
  begin(default_sda_pin, default_scl_pin);
}

void TwoWire::begin(uint8_t address){
  // twi_setAddress(address);
  // twi_attachSlaveTxEvent(onRequestService);
  // twi_attachSlaveRxEvent(onReceiveService);
  begin();
}

void TwoWire::begin(int address){
  begin((uint8_t)address);
}

void TwoWire::setClock(uint32_t frequency){
  twi_setClock(frequency);
}

size_t TwoWire::requestFrom(uint8_t address, size_t size, bool sendStop){
  if(size > BUFFER_LENGTH){
    size = BUFFER_LENGTH;
  }
  size_t read = (twi_readFrom(address, rxBuffer, size, sendStop) == 0)?size:0;
  rxBufferIndex = 0;
  rxBufferLength = read;
  return read;
}

uint8_t TwoWire::requestFrom(uint8_t address, uint8_t quantity, uint8_t sendStop){
  return requestFrom(address, static_cast<size_t>(quantity), static_cast<bool>(sendStop));
}

uint8_t TwoWire::requestFrom(uint8_t address, uint8_t quantity){
  return requestFrom(address, static_cast<size_t>(quantity), true);
}

uint8_t TwoWire::requestFrom(int address, int quantity){
  return requestFrom(static_cast<uint8_t>(address), static_cast<size_t>(quantity), true);
}

uint8_t TwoWire::requestFrom(int address, int quantity, int sendStop){
  return requestFrom(static_cast<uint8_t>(address), static_cast<size_t>(quantity), static_cast<bool>(sendStop));
}

void TwoWire::beginTransmission(uint8_t address){
  transmitting = 1;
  txAddress = address;
  txBufferIndex = 0;
  txBufferLength = 0;
}

void TwoWire::beginTransmission(int address){
  beginTransmission((uint8_t)address);
}

uint8_t TwoWire::endTransmission(uint8_t sendStop){
  int8_t ret = twi_writeTo(txAddress, txBuffer, txBufferLength, sendStop);
  txBufferIndex = 0;
  txBufferLength = 0;
  transmitting = 0;
  return ret;
}

uint8_t TwoWire::endTransmission(void){
  return endTransmission(true);
}

size_t TwoWire::write(uint8_t data){
  if(transmitting){
    if(txBufferLength >= BUFFER_LENGTH){
      setWriteError();
      return 0;
    }
    txBuffer[txBufferIndex] = data;
    ++txBufferIndex;
    txBufferLength = txBufferIndex;
  } else {
    // i2c_slave_transmit(&data, 1);
  }
  return 1;
}

size_t TwoWire::write(const uint8_t *data, size_t quantity){
  if(transmitting){
    for(size_t i = 0; i < quantity; ++i){
      if(!write(data[i])) return i;
    }
  }else{
    // i2c_slave_transmit(data, quantity);
  }
  return quantity;
}

int TwoWire::available(void){
  return rxBufferLength - rxBufferIndex;
}

int TwoWire::read(void){
  int value = -1;
  if(rxBufferIndex < rxBufferLength){
    value = rxBuffer[rxBufferIndex];
    ++rxBufferIndex;
  }
  return value;
}

int TwoWire::peek(void){
  int value = -1;
  if(rxBufferIndex < rxBufferLength){
    value = rxBuffer[rxBufferIndex];
  }
  return value;
}

void TwoWire::flush(void){
  rxBufferIndex = 0;
  rxBufferLength = 0;
  txBufferIndex = 0;
  txBufferLength = 0;
}

void TwoWire::onReceiveService(uint8_t* inBytes, int numBytes)
{
  // don't bother if user hasn't registered a callback
  // if(!user_onReceive){
  //   return;
  // }
  // // don't bother if rx buffer is in use by a master requestFrom() op
  // // i know this drops data, but it allows for slight stupidity
  // // meaning, they may not have read all the master requestFrom() data yet
  // if(rxBufferIndex < rxBufferLength){
  //   return;
  // }
  // // copy twi rx buffer into local read buffer
  // // this enables new reads to happen in parallel
  // for(uint8_t i = 0; i < numBytes; ++i){
  //   rxBuffer[i] = inBytes[i];
  // }
  // // set rx iterator vars
  // rxBufferIndex = 0;
  // rxBufferLength = numBytes;
  // // alert user program
  // user_onReceive(numBytes);
}

void TwoWire::onRequestService(void){
  // // don't bother if user hasn't registered a callback
  // if(!user_onRequest){
  //   return;
  // }
  // // reset tx buffer iterator vars
  // // !!! this will kill any pending pre-master sendTo() activity
  // txBufferIndex = 0;
  // txBufferLength = 0;
  // // alert user program
  // user_onRequest();
}

void TwoWire::onReceive( void (*function)(int) ){
  //user_onReceive = function;
}

void TwoWire::onRequest( void (*function)(void) ){
  //user_onRequest = function;
}

// Preinstantiate Objects //////////////////////////////////////////////////////

TwoWire Wire = TwoWire();

<|endoftext|>


<commit_before>#include <QThread>

int main()
{
    QThread::sleep(60);
}
<commit_msg>Fix "inifiniteLoop" test for Qt 4.<commit_after>#include <QThread>

class MyThread : public QThread
{
public:
    static void mySleep(unsigned long secs) { sleep(secs); } // sleep() is protected in Qt 4.
};

int main()
{
    MyThread::mySleep(60);
}
<|endoftext|>


<commit_before>// Copyright (c) 2020 by Robert Bosch GmbH. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "iceoryx_posh/internal/popo/waitset/condition.hpp"
#include "iceoryx_posh/internal/popo/waitset/condition_variable_data.hpp"
#include "iceoryx_posh/internal/popo/waitset/guard_condition.hpp"
#include "iceoryx_posh/internal/popo/waitset/wait_set.hpp"
#include "iceoryx_utils/cxx/vector.hpp"
#include "test.hpp"

#include <memory>

using namespace ::testing;
using ::testing::Return;
using namespace iox::popo;
using namespace iox::cxx;
using namespace iox::units::duration_literals;

class MockSubscriber : public Condition
{
  public:
    bool isConditionVariableAttached() noexcept override
    {
        return m_condVarAttached;
    }

    bool attachConditionVariable(ConditionVariableData* ConditionVariableDataPtr) noexcept override
    {
        m_condVarAttached = true;
        m_condVarPtr = ConditionVariableDataPtr;
        return m_condVarAttached;
    }

    bool hasTriggered() const noexcept override
    {
        return m_wasTriggered;
    }

    bool detachConditionVariable() noexcept override
    {
        m_condVarAttached = false;
        m_condVarPtr = nullptr;
        return m_condVarAttached;
    }

    /// @note done in ChunkQueuePusher
    void notify()
    {
        m_wasTriggered = true;
        ConditionVariableSignaler signaler{m_condVarPtr};
        signaler.notifyOne();
    }

    /// @note members reside in ChunkQueueData in SHM
    bool m_condVarAttached{false};
    bool m_wasTriggered{false};
    ConditionVariableData* m_condVarPtr{nullptr};
};

class WaitSet_test : public Test
{
  public:
    ConditionVariableData m_condVarData;
    WaitSet m_sut{&m_condVarData};
    vector<MockSubscriber, iox::MAX_NUMBER_OF_CONDITIONS> m_subscriberVector;

    iox::posix::Semaphore m_syncSemaphore = iox::posix::Semaphore::create(0u).get_value();

    void SetUp()
    {
        MockSubscriber subscriber;
        while (m_subscriberVector.size() != m_subscriberVector.capacity())
        {
            m_subscriberVector.push_back(subscriber);
        }
    };

    void TearDown()
    {
        m_subscriberVector.clear();
        m_sut.clear();
        ConditionVariableWaiter waiter{&m_condVarData};
        waiter.reset();
    };
};

TEST_F(WaitSet_test, AttachSingleConditionSuccessful)
{
    EXPECT_TRUE(m_sut.attachCondition(m_subscriberVector.front()));
}

TEST_F(WaitSet_test, AttachSameConditionTwiceResultsInFailure)
{
    m_sut.attachCondition(m_subscriberVector.front());
    EXPECT_FALSE(m_sut.attachCondition(m_subscriberVector.front()));
}

TEST_F(WaitSet_test, AttachMultipleConditionSuccessful)
{
    for (auto& currentSubscriber : m_subscriberVector)
    {
        EXPECT_TRUE(m_sut.attachCondition(currentSubscriber));
    }
}

TEST_F(WaitSet_test, AttachTooManyConditionsResultsInFailure)
{
    for (auto& currentSubscriber : m_subscriberVector)
    {
        m_sut.attachCondition(currentSubscriber);
    }

    MockSubscriber extraCondition;
    EXPECT_FALSE(m_sut.attachCondition(extraCondition));
}

TEST_F(WaitSet_test, DetachSingleConditionSuccessful)
{
    m_sut.attachCondition(m_subscriberVector.front());
    EXPECT_TRUE(m_sut.detachCondition(m_subscriberVector.front()));
}

TEST_F(WaitSet_test, DetachMultipleleConditionsSuccessful)
{
    for (auto& currentSubscriber : m_subscriberVector)
    {
        m_sut.attachCondition(currentSubscriber);
    }
    for (auto& currentSubscriber : m_subscriberVector)
    {
        EXPECT_TRUE(m_sut.detachCondition(currentSubscriber));
    }
}

TEST_F(WaitSet_test, DetachConditionNotInListResultsInFailure)
{
    EXPECT_FALSE(m_sut.detachCondition(m_subscriberVector.front()));
}

TEST_F(WaitSet_test, DetachUnknownConditionResultsInFailure)
{
    m_sut.attachCondition(m_subscriberVector.front());
    EXPECT_FALSE(m_sut.detachCondition(m_subscriberVector.back()));
}

TEST_F(WaitSet_test, TimedWaitWithInvalidTimeResultsInEmptyVector)
{
    auto emptyVector = m_sut.timedWait(0_ms);
    EXPECT_TRUE(emptyVector.empty());
}

TEST_F(WaitSet_test, NoAttachTimedWaitResultsInEmptyVector)
{
    auto emptyVector = m_sut.timedWait(1_ms);
    EXPECT_TRUE(emptyVector.empty());
}

TEST_F(WaitSet_test, TimedWaitWithNotificationResultsInImmediateTrigger)
{
    m_sut.attachCondition(m_subscriberVector.front());
    m_subscriberVector.front().notify();
    auto fulfilledConditions = m_sut.timedWait(1_ms);
    EXPECT_THAT(fulfilledConditions.size(), Eq(1));
    EXPECT_THAT(fulfilledConditions.front(), &m_subscriberVector.front());
}

TEST_F(WaitSet_test, TimeoutOfTimedWaitResultsInEmptyVector)
{
    m_sut.attachCondition(m_subscriberVector.front());
    auto fulfilledConditions = m_sut.timedWait(1_ms);
    EXPECT_THAT(fulfilledConditions.size(), Eq(0));
}

TEST_F(WaitSet_test, NotifyOneWhileWaitingResultsInTriggerMultiThreaded)
{
    std::atomic<int> counter{0};
    m_sut.attachCondition(m_subscriberVector.front());
    std::thread waiter([&] {
        EXPECT_THAT(counter, Eq(0));
        m_syncSemaphore.post();
        auto fulfilledConditions = m_sut.wait();
        EXPECT_THAT(fulfilledConditions.size(), Eq(1));
        EXPECT_THAT(fulfilledConditions.front(), &m_subscriberVector.front());
        EXPECT_THAT(counter, Eq(1));
    });
    m_syncSemaphore.wait();
    counter++;
    m_subscriberVector.front().notify();
    waiter.join();
}

TEST_F(WaitSet_test, AttachManyNotifyOneWhileWaitingResultsInTriggerMultiThreaded)
{
    std::atomic<int> counter{0};
    m_sut.attachCondition(m_subscriberVector[0]);
    m_sut.attachCondition(m_subscriberVector[1]);
    std::thread waiter([&] {
        EXPECT_THAT(counter, Eq(0));
        m_syncSemaphore.post();
        auto fulfilledConditions = m_sut.wait();
        EXPECT_THAT(fulfilledConditions.size(), Eq(1));
        EXPECT_THAT(fulfilledConditions.front(), &m_subscriberVector[0]);
        EXPECT_THAT(counter, Eq(1));
    });
    m_syncSemaphore.wait();
    counter++;
    m_subscriberVector[0].notify();
    waiter.join();
}

TEST_F(WaitSet_test, AttachManyNotifyManyBeforeWaitingResultsInTriggerMultiThreaded)
{
    std::atomic<int> counter{0};
    m_sut.attachCondition(m_subscriberVector[0]);
    m_sut.attachCondition(m_subscriberVector[1]);
    std::thread waiter([&] {
        EXPECT_THAT(counter, Eq(0));
        m_syncSemaphore.post();
        std::this_thread::sleep_for(std::chrono::milliseconds(1));
        m_syncSemaphore.wait();
        auto fulfilledConditions = m_sut.wait();
        EXPECT_THAT(fulfilledConditions.size(), Eq(2));
        EXPECT_THAT(fulfilledConditions[0], &m_subscriberVector[0]);
        EXPECT_THAT(fulfilledConditions[1], &m_subscriberVector[1]);
        EXPECT_THAT(counter, Eq(1));
    });
    m_syncSemaphore.wait();
    m_subscriberVector[0].notify();
    m_subscriberVector[1].notify();
    counter++;
    m_syncSemaphore.post();
    waiter.join();
}

TEST_F(WaitSet_test, AttachManyNotifyManyWhileWaitingResultsInTriggerMultiThreaded)
{
    std::atomic<int> counter{0};
    m_sut.attachCondition(m_subscriberVector[0]);
    m_sut.attachCondition(m_subscriberVector[1]);
    std::thread waiter([&] {
        EXPECT_THAT(counter, Eq(0));
        m_syncSemaphore.post();
        auto fulfilledConditions = m_sut.wait();
        EXPECT_THAT(fulfilledConditions.size(), Eq(2));
        EXPECT_THAT(fulfilledConditions[0], &m_subscriberVector[0]);
        EXPECT_THAT(fulfilledConditions[1], &m_subscriberVector[1]);
        EXPECT_THAT(counter, Eq(1));
    });
    m_syncSemaphore.wait();
    m_subscriberVector[0].notify();
    m_subscriberVector[1].notify();
    counter++;
    waiter.join();
}

TEST_F(WaitSet_test, WaitWithoutNotifyResultsInBlockingMultiThreaded)
{
    std::atomic<int> counter{0};
    m_sut.attachCondition(m_subscriberVector.front());
    std::thread waiter([&] {
        EXPECT_THAT(counter, Eq(0));
        m_syncSemaphore.post();
        m_sut.wait();
        EXPECT_THAT(counter, Eq(1));
    });
    m_syncSemaphore.wait();
    counter++;
    m_subscriberVector.front().notify();
    waiter.join();
}

TEST_F(WaitSet_test, NotifyGuardConditionWhileWaitingResultsInTriggerMultiThreaded)
{
    std::atomic<int> counter{0};
    GuardCondition guardCond;
    m_sut.attachCondition(guardCond);
    std::thread waiter([&] {
        EXPECT_THAT(counter, Eq(0));
        m_syncSemaphore.post();
        auto fulfilledConditions = m_sut.wait();
        EXPECT_THAT(fulfilledConditions.size(), Eq(1));
        EXPECT_THAT(fulfilledConditions.front(), &guardCond);
        EXPECT_THAT(counter, Eq(1));
    });
    m_syncSemaphore.wait();
    counter++;
    guardCond.notify();
    waiter.join();
}

TEST_F(WaitSet_test, NotifyGuardConditionOnceTimedWaitResultsInResetOfTrigger)
{
    GuardCondition guardCond;
    m_sut.attachCondition(guardCond);
    guardCond.notify();
    auto fulfilledConditions1 = m_sut.timedWait(1_ms);
    EXPECT_THAT(fulfilledConditions1.size(), Eq(1));
    EXPECT_THAT(fulfilledConditions1.front(), &guardCond);
    guardCond.resetTrigger();
    auto fulfilledConditions2 = m_sut.timedWait(1_ms);
    EXPECT_THAT(fulfilledConditions2.size(), Eq(0));
}
<commit_msg>iox-#25 Add further test case for WaitSet<commit_after>// Copyright (c) 2020 by Robert Bosch GmbH. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "iceoryx_posh/internal/popo/waitset/condition.hpp"
#include "iceoryx_posh/internal/popo/waitset/condition_variable_data.hpp"
#include "iceoryx_posh/internal/popo/waitset/guard_condition.hpp"
#include "iceoryx_posh/internal/popo/waitset/wait_set.hpp"
#include "iceoryx_utils/cxx/vector.hpp"
#include "test.hpp"

#include <memory>

using namespace ::testing;
using ::testing::Return;
using namespace iox::popo;
using namespace iox::cxx;
using namespace iox::units::duration_literals;

class MockSubscriber : public Condition
{
  public:
    bool isConditionVariableAttached() noexcept override
    {
        return m_condVarAttached;
    }

    bool attachConditionVariable(ConditionVariableData* ConditionVariableDataPtr) noexcept override
    {
        m_condVarAttached = true;
        m_condVarPtr = ConditionVariableDataPtr;
        return m_condVarAttached;
    }

    bool hasTriggered() const noexcept override
    {
        return m_wasTriggered;
    }

    bool detachConditionVariable() noexcept override
    {
        m_condVarAttached = false;
        m_condVarPtr = nullptr;
        return m_condVarAttached;
    }

    /// @note done in ChunkQueuePusher
    void notify()
    {
        m_wasTriggered = true;
        ConditionVariableSignaler signaler{m_condVarPtr};
        signaler.notifyOne();
    }

    /// @note members reside in ChunkQueueData in SHM
    bool m_condVarAttached{false};
    bool m_wasTriggered{false};
    ConditionVariableData* m_condVarPtr{nullptr};
};

class WaitSet_test : public Test
{
  public:
    ConditionVariableData m_condVarData;
    WaitSet m_sut{&m_condVarData};
    vector<MockSubscriber, iox::MAX_NUMBER_OF_CONDITIONS> m_subscriberVector;

    iox::posix::Semaphore m_syncSemaphore = iox::posix::Semaphore::create(0u).get_value();

    void SetUp()
    {
        MockSubscriber subscriber;
        while (m_subscriberVector.size() != m_subscriberVector.capacity())
        {
            m_subscriberVector.push_back(subscriber);
        }
    };

    void TearDown()
    {
        m_subscriberVector.clear();
        m_sut.clear();
        ConditionVariableWaiter waiter{&m_condVarData};
        waiter.reset();
    };
};

TEST_F(WaitSet_test, AttachSingleConditionSuccessful)
{
    EXPECT_TRUE(m_sut.attachCondition(m_subscriberVector.front()));
}

TEST_F(WaitSet_test, AttachSameConditionTwiceResultsInFailure)
{
    m_sut.attachCondition(m_subscriberVector.front());
    EXPECT_FALSE(m_sut.attachCondition(m_subscriberVector.front()));
}

TEST_F(WaitSet_test, AttachMultipleConditionSuccessful)
{
    for (auto& currentSubscriber : m_subscriberVector)
    {
        EXPECT_TRUE(m_sut.attachCondition(currentSubscriber));
    }
}

TEST_F(WaitSet_test, AttachTooManyConditionsResultsInFailure)
{
    for (auto& currentSubscriber : m_subscriberVector)
    {
        m_sut.attachCondition(currentSubscriber);
    }

    MockSubscriber extraCondition;
    EXPECT_FALSE(m_sut.attachCondition(extraCondition));
}

TEST_F(WaitSet_test, DetachSingleConditionSuccessful)
{
    m_sut.attachCondition(m_subscriberVector.front());
    EXPECT_TRUE(m_sut.detachCondition(m_subscriberVector.front()));
}

TEST_F(WaitSet_test, DetachMultipleleConditionsSuccessful)
{
    for (auto& currentSubscriber : m_subscriberVector)
    {
        m_sut.attachCondition(currentSubscriber);
    }
    for (auto& currentSubscriber : m_subscriberVector)
    {
        EXPECT_TRUE(m_sut.detachCondition(currentSubscriber));
    }
}

TEST_F(WaitSet_test, DetachConditionNotInListResultsInFailure)
{
    EXPECT_FALSE(m_sut.detachCondition(m_subscriberVector.front()));
}

TEST_F(WaitSet_test, DetachUnknownConditionResultsInFailure)
{
    m_sut.attachCondition(m_subscriberVector.front());
    EXPECT_FALSE(m_sut.detachCondition(m_subscriberVector.back()));
}

TEST_F(WaitSet_test, TimedWaitWithInvalidTimeResultsInEmptyVector)
{
    auto emptyVector = m_sut.timedWait(0_ms);
    EXPECT_TRUE(emptyVector.empty());
}

TEST_F(WaitSet_test, NoAttachTimedWaitResultsInEmptyVector)
{
    auto emptyVector = m_sut.timedWait(1_ms);
    EXPECT_TRUE(emptyVector.empty());
}

TEST_F(WaitSet_test, TimedWaitWithMaximumNumberOfConditionsResultsInReturnOfMaximumNumberOfConditions)
{
    for (auto& currentSubscriber : m_subscriberVector)
    {
        m_sut.attachCondition(currentSubscriber);
        currentSubscriber.notify();
    }
    auto fulfilledConditions = m_sut.timedWait(1_ms);
    EXPECT_THAT(fulfilledConditions.size(), Eq(iox::MAX_NUMBER_OF_CONDITIONS));
}

TEST_F(WaitSet_test, TimedWaitWithNotificationResultsInImmediateTrigger)
{
    m_sut.attachCondition(m_subscriberVector.front());
    m_subscriberVector.front().notify();
    auto fulfilledConditions = m_sut.timedWait(1_ms);
    EXPECT_THAT(fulfilledConditions.size(), Eq(1));
    EXPECT_THAT(fulfilledConditions.front(), &m_subscriberVector.front());
}

TEST_F(WaitSet_test, TimeoutOfTimedWaitResultsInEmptyVector)
{
    m_sut.attachCondition(m_subscriberVector.front());
    auto fulfilledConditions = m_sut.timedWait(1_ms);
    EXPECT_THAT(fulfilledConditions.size(), Eq(0));
}

TEST_F(WaitSet_test, NotifyOneWhileWaitingResultsInTriggerMultiThreaded)
{
    std::atomic<int> counter{0};
    m_sut.attachCondition(m_subscriberVector.front());
    std::thread waiter([&] {
        EXPECT_THAT(counter, Eq(0));
        m_syncSemaphore.post();
        auto fulfilledConditions = m_sut.wait();
        EXPECT_THAT(fulfilledConditions.size(), Eq(1));
        EXPECT_THAT(fulfilledConditions.front(), &m_subscriberVector.front());
        EXPECT_THAT(counter, Eq(1));
    });
    m_syncSemaphore.wait();
    counter++;
    m_subscriberVector.front().notify();
    waiter.join();
}

TEST_F(WaitSet_test, AttachManyNotifyOneWhileWaitingResultsInTriggerMultiThreaded)
{
    std::atomic<int> counter{0};
    m_sut.attachCondition(m_subscriberVector[0]);
    m_sut.attachCondition(m_subscriberVector[1]);
    std::thread waiter([&] {
        EXPECT_THAT(counter, Eq(0));
        m_syncSemaphore.post();
        auto fulfilledConditions = m_sut.wait();
        EXPECT_THAT(fulfilledConditions.size(), Eq(1));
        EXPECT_THAT(fulfilledConditions.front(), &m_subscriberVector[0]);
        EXPECT_THAT(counter, Eq(1));
    });
    m_syncSemaphore.wait();
    counter++;
    m_subscriberVector[0].notify();
    waiter.join();
}

TEST_F(WaitSet_test, AttachManyNotifyManyBeforeWaitingResultsInTriggerMultiThreaded)
{
    std::atomic<int> counter{0};
    m_sut.attachCondition(m_subscriberVector[0]);
    m_sut.attachCondition(m_subscriberVector[1]);
    std::thread waiter([&] {
        EXPECT_THAT(counter, Eq(0));
        m_syncSemaphore.post();
        std::this_thread::sleep_for(std::chrono::milliseconds(1));
        m_syncSemaphore.wait();
        auto fulfilledConditions = m_sut.wait();
        EXPECT_THAT(fulfilledConditions.size(), Eq(2));
        EXPECT_THAT(fulfilledConditions[0], &m_subscriberVector[0]);
        EXPECT_THAT(fulfilledConditions[1], &m_subscriberVector[1]);
        EXPECT_THAT(counter, Eq(1));
    });
    m_syncSemaphore.wait();
    m_subscriberVector[0].notify();
    m_subscriberVector[1].notify();
    counter++;
    m_syncSemaphore.post();
    waiter.join();
}

TEST_F(WaitSet_test, AttachManyNotifyManyWhileWaitingResultsInTriggerMultiThreaded)
{
    std::atomic<int> counter{0};
    m_sut.attachCondition(m_subscriberVector[0]);
    m_sut.attachCondition(m_subscriberVector[1]);
    std::thread waiter([&] {
        EXPECT_THAT(counter, Eq(0));
        m_syncSemaphore.post();
        auto fulfilledConditions = m_sut.wait();
        EXPECT_THAT(fulfilledConditions.size(), Eq(2));
        EXPECT_THAT(fulfilledConditions[0], &m_subscriberVector[0]);
        EXPECT_THAT(fulfilledConditions[1], &m_subscriberVector[1]);
        EXPECT_THAT(counter, Eq(1));
    });
    m_syncSemaphore.wait();
    m_subscriberVector[0].notify();
    m_subscriberVector[1].notify();
    counter++;
    waiter.join();
}

TEST_F(WaitSet_test, WaitWithoutNotifyResultsInBlockingMultiThreaded)
{
    std::atomic<int> counter{0};
    m_sut.attachCondition(m_subscriberVector.front());
    std::thread waiter([&] {
        EXPECT_THAT(counter, Eq(0));
        m_syncSemaphore.post();
        m_sut.wait();
        EXPECT_THAT(counter, Eq(1));
    });
    m_syncSemaphore.wait();
    counter++;
    m_subscriberVector.front().notify();
    waiter.join();
}

TEST_F(WaitSet_test, NotifyGuardConditionWhileWaitingResultsInTriggerMultiThreaded)
{
    std::atomic<int> counter{0};
    GuardCondition guardCond;
    m_sut.attachCondition(guardCond);
    std::thread waiter([&] {
        EXPECT_THAT(counter, Eq(0));
        m_syncSemaphore.post();
        auto fulfilledConditions = m_sut.wait();
        EXPECT_THAT(fulfilledConditions.size(), Eq(1));
        EXPECT_THAT(fulfilledConditions.front(), &guardCond);
        EXPECT_THAT(counter, Eq(1));
    });
    m_syncSemaphore.wait();
    counter++;
    guardCond.notify();
    waiter.join();
}

TEST_F(WaitSet_test, NotifyGuardConditionOnceTimedWaitResultsInResetOfTrigger)
{
    GuardCondition guardCond;
    m_sut.attachCondition(guardCond);
    guardCond.notify();
    auto fulfilledConditions1 = m_sut.timedWait(1_ms);
    EXPECT_THAT(fulfilledConditions1.size(), Eq(1));
    EXPECT_THAT(fulfilledConditions1.front(), &guardCond);
    guardCond.resetTrigger();
    auto fulfilledConditions2 = m_sut.timedWait(1_ms);
    EXPECT_THAT(fulfilledConditions2.size(), Eq(0));
}
<|endoftext|>


<commit_before>#pragma once

#include <unordered_map>
#include <vector>

#include "depthai-shared/datatype/RawCameraControl.hpp"
#include "depthai/pipeline/datatype/Buffer.hpp"

namespace dai {

// protected inheritance, so serialize isn't visible to users
class CameraControl : public Buffer {
    std::shared_ptr<RawBuffer> serialize() const override;
    RawCameraControl& cfg;

    using AutoFocusMode = RawCameraControl::AutoFocusMode;
    using AntiBandingMode = RawCameraControl::AntiBandingMode;
    using AutoWhiteBalanceMode = RawCameraControl::AutoWhiteBalanceMode;
    using SceneMode = RawCameraControl::SceneMode;
    using EffectMode = RawCameraControl::EffectMode;

   public:
    CameraControl();
    explicit CameraControl(std::shared_ptr<RawCameraControl> ptr);
    virtual ~CameraControl() = default;

    // Functions to set properties
    void setCaptureStill(bool capture);

    void setStartStreaming();
    void setStopStreaming();

    // Focus
    void setAutoFocusMode(AutoFocusMode mode);
    void setAutoFocusTrigger();
    void setAutoFocusRegion(uint16_t startX, uint16_t startY, uint16_t width, uint16_t height);
    void setManualFocus(uint8_t lensPosition);

    // Exposure
    void setAutoExposureEnable();
    void setAutoExposureLock(bool lock);
    void setAutoExposureRegion(uint16_t startX, uint16_t startY, uint16_t width, uint16_t height);
    void setAutoExposureCompensation(int8_t compensation);
    void setAntiBandingMode(AntiBandingMode mode);
    void setManualExposure(uint32_t exposureTimeUs, uint32_t sensitivityIso);

    // White Balance
    void setAutoWhiteBalanceMode(AutoWhiteBalanceMode mode);
    void setAutoWhiteBalanceLock(bool lock);

    // Other image controls
    void setBrightness(uint16_t value);  // TODO move to AE?
    void setContrast(uint16_t value);
    void setSaturation(uint16_t value);
    void setSharpness(uint16_t value);
    void setNoiseReductionStrength(uint16_t value);
    void setLumaDenoise(uint16_t value);
    void setChromaDenoise(uint16_t value);
    void setSceneMode(SceneMode mode);
    void setEffectMode(EffectMode mode);

    // Functions to retrieve properties
    bool getCaptureStill() const;
};

}  // namespace dai
<commit_msg>CameraControl.hpp: make non-Raw types public<commit_after>#pragma once

#include <unordered_map>
#include <vector>

#include "depthai-shared/datatype/RawCameraControl.hpp"
#include "depthai/pipeline/datatype/Buffer.hpp"

namespace dai {

// protected inheritance, so serialize isn't visible to users
class CameraControl : public Buffer {
    std::shared_ptr<RawBuffer> serialize() const override;
    RawCameraControl& cfg;

   public:
    using AutoFocusMode = RawCameraControl::AutoFocusMode;
    using AntiBandingMode = RawCameraControl::AntiBandingMode;
    using AutoWhiteBalanceMode = RawCameraControl::AutoWhiteBalanceMode;
    using SceneMode = RawCameraControl::SceneMode;
    using EffectMode = RawCameraControl::EffectMode;

    CameraControl();
    explicit CameraControl(std::shared_ptr<RawCameraControl> ptr);
    virtual ~CameraControl() = default;

    // Functions to set properties
    void setCaptureStill(bool capture);

    void setStartStreaming();
    void setStopStreaming();

    // Focus
    void setAutoFocusMode(AutoFocusMode mode);
    void setAutoFocusTrigger();
    void setAutoFocusRegion(uint16_t startX, uint16_t startY, uint16_t width, uint16_t height);
    void setManualFocus(uint8_t lensPosition);

    // Exposure
    void setAutoExposureEnable();
    void setAutoExposureLock(bool lock);
    void setAutoExposureRegion(uint16_t startX, uint16_t startY, uint16_t width, uint16_t height);
    void setAutoExposureCompensation(int8_t compensation);
    void setAntiBandingMode(AntiBandingMode mode);
    void setManualExposure(uint32_t exposureTimeUs, uint32_t sensitivityIso);

    // White Balance
    void setAutoWhiteBalanceMode(AutoWhiteBalanceMode mode);
    void setAutoWhiteBalanceLock(bool lock);

    // Other image controls
    void setBrightness(uint16_t value);  // TODO move to AE?
    void setContrast(uint16_t value);
    void setSaturation(uint16_t value);
    void setSharpness(uint16_t value);
    void setNoiseReductionStrength(uint16_t value);
    void setLumaDenoise(uint16_t value);
    void setChromaDenoise(uint16_t value);
    void setSceneMode(SceneMode mode);
    void setEffectMode(EffectMode mode);

    // Functions to retrieve properties
    bool getCaptureStill() const;
};

}  // namespace dai
<|endoftext|>


<commit_before>// @(#)root/proof:$Id$
// Author: G. Ganis, Mar 2010

/*************************************************************************
 * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

//////////////////////////////////////////////////////////////////////////
//                                                                      //
// pq2wrappers                                                          //
//                                                                      //
// This file implements the wrapper functions used in PQ2               //
//                                                                      //
//////////////////////////////////////////////////////////////////////////
#include <stdlib.h>

#include "pq2wrappers.h"
#include "redirguard.h"

#include "TDataSetManager.h"
#include "TEnv.h"
#include "TFileInfo.h"
#include "TPluginManager.h"
#include "TProof.h"
#include "TROOT.h"

TDataSetManager *gDataSetManager = 0;

// Global variables defined by other PQ2 components
extern TUrl    gUrl;
extern Bool_t  gIsProof;
extern TString flog;
extern TString ferr;
extern TString fres;
extern Int_t gverbose;

Int_t getProof(const char *where, Int_t verbose = 1);
Int_t getDSMgr(const char *where);

//_______________________________________________________________________________________
void DataSetCache(bool clear, const char *ds)
{
   // ShowCache wrapper
   if (gIsProof) {
      if (!gProof && getProof("DataSetCache", 0) != 0) return;
      return (clear ? gProof->ClearDataSetCache(ds) : gProof->ShowDataSetCache(ds));
   } else {
      if (!gDataSetManager && getDSMgr("DataSetCache") != 0) return;
      Int_t rc = (clear) ? gDataSetManager->ClearCache(ds) : gDataSetManager->ShowCache(ds);
      if (rc != 0)
         Printf("DataSetCache: problems running '%s'", (clear ? "clear" : "show"));
      return;
   }
   // Done
   return;
}

//_______________________________________________________________________________________
void ShowDataSets(const char *ds, const char *opt)
{
   // ShowDataSets wrapper
   if (gIsProof) {
      if (!gProof && getProof("ShowDataSets", 0) != 0) return;
      return gProof->ShowDataSets(ds, opt);
   } else {
      if (!gDataSetManager && getDSMgr("ShowDataSets") != 0) return;
      return gDataSetManager->ShowDataSets(ds, opt);
   }
   // Done
   return;
}

//_______________________________________________________________________________________
TFileCollection *GetDataSet(const char *ds, const char *server)
{
   // GetDataSet wrapper
   TFileCollection *fc = 0;
   if (gIsProof) {
      if (!gProof && getProof("GetDataSet") != 0) return fc;
      return gProof->GetDataSet(ds, server);
   } else {
      if (!gDataSetManager && getDSMgr("ShowDataSets") != 0) return fc;
      return gDataSetManager->GetDataSet(ds, server);
   }
   // Done
   return fc;
}

//_______________________________________________________________________________________
TMap *GetDataSets(const char *owner, const char *server, const char *opt)
{
   // GetDataSets wrapper
   TMap *dss = 0;
   if (gIsProof) {
      if (!gProof && getProof("GetDataSets") != 0) return dss;
      return gProof->GetDataSets(owner, server);
   } else {
      if (!gDataSetManager && getDSMgr("GetDataSets") != 0) return dss;
      // Get the datasets and fill a map
      UInt_t oo = (opt && !strcmp(opt, "list")) ? (UInt_t)TDataSetManager::kList
                                                : (UInt_t)TDataSetManager::kExport;
      dss = gDataSetManager->GetDataSets(owner, oo);
      // If defines, option gives the name of a server for which to extract the information
      if (dss) {
         if (server && strlen(server) > 0) {
            // The return map will be in the form   </group/user/datasetname> --> <dataset>
            TMap *rmap = new TMap;
            TObject *k = 0;
            TFileCollection *fc = 0, *xfc = 0;
            TIter nxd(dss);
            while ((k = nxd()) && (fc = (TFileCollection *) dss->GetValue(k))) {
               // Get subset on specified server, if any
               if ((xfc = fc->GetFilesOnServer(server))) {
                  rmap->Add(new TObjString(k->GetName()), xfc);
               }
            }
            dss->DeleteAll();
            delete dss;
            dss = 0;
            if (rmap->GetSize() > 0) {
               dss = rmap;
            } else {
               Printf("GetDataSets: no dataset found on server '%s' for owner '%s'", server, owner);
               delete rmap;
            }
         }
      } else  {
         Printf("GetDataSets: no dataset found for '%s'", owner);
      }
   }
   // Done
   return dss;
}

//_______________________________________________________________________________________
Int_t RemoveDataSet(const char *dsname)
{
   // RemoveDataSet wrapper
   if (gIsProof) {
      if (!gProof && getProof("RemoveDataSet") != 0) return -1;
      return gProof->RemoveDataSet(dsname);
   } else {
      if (!gDataSetManager && getDSMgr("RemoveDataSet") != 0) return -1;
      return gDataSetManager->RemoveDataSet(dsname);
   }
   // Done
   return -1;
}

//_______________________________________________________________________________________
Int_t VerifyDataSet(const char *dsname, const char *opt, const char *redir)
{
   // VerifyDataSet wrapper

   Int_t rc = -1;
   // Honour the 'redir' if required
   TString srvmaps;
   if (redir && strlen(redir) > 0) srvmaps.Form("|%s", redir);
   if (gIsProof) {
      // Honour the 'redir' if required
      if (!(srvmaps.IsNull())) {
         TProof::AddEnvVar("DATASETSRVMAPS", srvmaps);
      }
      if (!gProof && getProof("VerifyDataSet") != 0) return -1;
      if ((rc = gProof->VerifyDataSet(dsname, opt)) == 0) {
         // Success; partial at least. Check if all files are staged
         TFileCollection *fcs = gProof->GetDataSet(dsname, "S:");
         if (fcs && fcs->GetStagedPercentage() < 99.99999) rc = 1;
      }
   } else {
      // Honour the 'redir' if required
      if (!(srvmaps.IsNull())) {
         gEnv->SetValue("DataSet.SrvMaps", srvmaps);
      }
      if (!gDataSetManager && getDSMgr("VerifyDataSet") != 0) return -1;
      if ((rc = gDataSetManager->ScanDataSet(dsname, opt)) == 0) {
         // Success; partial at least. Check if all files are staged
         TFileCollection *fcs = gDataSetManager->GetDataSet(dsname, "S:");
         if (fcs && fcs->GetStagedPercentage() < 99.99999) rc = 1;
      }
   }
   // Done
   return rc;
}

//_______________________________________________________________________________________
Bool_t ExistsDataSet(const char *dsname)
{
   // ExistsDataSet wrapper
   if (gIsProof) {
      if (!gProof && getProof("ExistsDataSet") != 0) return kFALSE;
      return gProof->ExistsDataSet(dsname);
   } else {
      if (!gDataSetManager && getDSMgr("ExistsDataSet") != 0) return kFALSE;
      return gDataSetManager->ExistsDataSet(dsname);
   }
   return kFALSE;
}

//_______________________________________________________________________________________
Int_t RegisterDataSet(const char *dsname, TFileCollection *fc, const char* opt)
{
   // RegisterDataSet wrapper
   if (gIsProof) {
      if (!gProof && getProof("GetDataSet") != 0) return -1;
      return gProof->RegisterDataSet(dsname, fc, opt);
   } else {
      if (!gDataSetManager && getDSMgr("RegisterDataSet") != 0) return -1;
      return gDataSetManager->RegisterDataSet(dsname, fc, opt);
   }
   return -1;
}

//_______________________________________________________________________________________
Int_t getProof(const char *where, Int_t verbose)
{
   // Open a PROOF session at gUrl

   {  redirguard rog(flog.Data(), "a", verbose);
      TProof::Open(gUrl.GetUrl(), "masteronly");
   }
   if (!gProof || !gProof->IsValid()) {
      Printf("getProof:%s: problems starting a PROOF session at '%s'", where, gUrl.GetUrl());
      return -1;
   }
   if (gverbose >= 2) gProof->SetLogLevel(2);
   // Done
   return 0;
}

//_______________________________________________________________________________________
Int_t getDSMgr(const char *where)
{
   // Open a dataset manager for gUrl

   Int_t rc = -1;
   if (gROOT->GetPluginManager()) {
      // Find the appropriate handler
      TPluginHandler *h = gROOT->GetPluginManager()->FindHandler("TDataSetManager", "file");
      if (h && h->LoadPlugin() != -1) {
         TString group(getenv("PQ2GROUP")), user(getenv("PQ2USER"));
         TString dsm, opt("opt:-Ar:-Av:");
         const char *o = getenv("PQ2DSMGROPTS");
         if (o) {
            opt = "";
            if (strlen(o) > 0) opt.Form("opt:%s", o);
         }
         dsm.Form("file dir:%s %s", gUrl.GetUrl(), opt.Data());
         gDataSetManager = reinterpret_cast<TDataSetManager*>(h->ExecPlugin(3,
                                                              group.Data(), user.Data(),
                                                              dsm.Data()));
         if (gDataSetManager) {
            rc = 0;
         } else {
            Printf("getDSMgr:%s: problems creating a dataset manager at '%s'", where, gUrl.GetUrl());
         }
      }
   }
   // Done
   return rc;
}
<commit_msg>Make sure that pq2-verify copes well with parallel verification (Savannah #99603)<commit_after>// @(#)root/proof:$Id$
// Author: G. Ganis, Mar 2010

/*************************************************************************
 * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

//////////////////////////////////////////////////////////////////////////
//                                                                      //
// pq2wrappers                                                          //
//                                                                      //
// This file implements the wrapper functions used in PQ2               //
//                                                                      //
//////////////////////////////////////////////////////////////////////////
#include <stdlib.h>

#include "pq2wrappers.h"
#include "redirguard.h"

#include "TDataSetManager.h"
#include "TEnv.h"
#include "TFileInfo.h"
#include "TPluginManager.h"
#include "TProof.h"
#include "TROOT.h"

TDataSetManager *gDataSetManager = 0;

// Global variables defined by other PQ2 components
extern TUrl    gUrl;
extern Bool_t  gIsProof;
extern TString flog;
extern TString ferr;
extern TString fres;
extern Int_t gverbose;

// How to start PROOF (new VerifyDataSet wants it parallel)
static bool doParallel = kFALSE;

Int_t getProof(const char *where, Int_t verbose = 1);
Int_t getDSMgr(const char *where);

//_______________________________________________________________________________________
void DataSetCache(bool clear, const char *ds)
{
   // ShowCache wrapper
   if (gIsProof) {
      doParallel = kFALSE;
      if (!gProof && getProof("DataSetCache", 0) != 0) return;
      return (clear ? gProof->ClearDataSetCache(ds) : gProof->ShowDataSetCache(ds));
   } else {
      if (!gDataSetManager && getDSMgr("DataSetCache") != 0) return;
      Int_t rc = (clear) ? gDataSetManager->ClearCache(ds) : gDataSetManager->ShowCache(ds);
      if (rc != 0)
         Printf("DataSetCache: problems running '%s'", (clear ? "clear" : "show"));
      return;
   }
   // Done
   return;
}

//_______________________________________________________________________________________
void ShowDataSets(const char *ds, const char *opt)
{
   // ShowDataSets wrapper
   if (gIsProof) {
      doParallel = kFALSE;
      if (!gProof && getProof("ShowDataSets", 0) != 0) return;
      return gProof->ShowDataSets(ds, opt);
   } else {
      if (!gDataSetManager && getDSMgr("ShowDataSets") != 0) return;
      return gDataSetManager->ShowDataSets(ds, opt);
   }
   // Done
   return;
}

//_______________________________________________________________________________________
TFileCollection *GetDataSet(const char *ds, const char *server)
{
   // GetDataSet wrapper
   TFileCollection *fc = 0;
   if (gIsProof) {
      doParallel = kFALSE;
      if (!gProof && getProof("GetDataSet") != 0) return fc;
      return gProof->GetDataSet(ds, server);
   } else {
      if (!gDataSetManager && getDSMgr("ShowDataSets") != 0) return fc;
      return gDataSetManager->GetDataSet(ds, server);
   }
   // Done
   return fc;
}

//_______________________________________________________________________________________
TMap *GetDataSets(const char *owner, const char *server, const char *opt)
{
   // GetDataSets wrapper
   TMap *dss = 0;
   if (gIsProof) {
      doParallel = kFALSE;
      if (!gProof && getProof("GetDataSets") != 0) return dss;
      return gProof->GetDataSets(owner, server);
   } else {
      if (!gDataSetManager && getDSMgr("GetDataSets") != 0) return dss;
      // Get the datasets and fill a map
      UInt_t oo = (opt && !strcmp(opt, "list")) ? (UInt_t)TDataSetManager::kList
                                                : (UInt_t)TDataSetManager::kExport;
      dss = gDataSetManager->GetDataSets(owner, oo);
      // If defines, option gives the name of a server for which to extract the information
      if (dss) {
         if (server && strlen(server) > 0) {
            // The return map will be in the form   </group/user/datasetname> --> <dataset>
            TMap *rmap = new TMap;
            TObject *k = 0;
            TFileCollection *fc = 0, *xfc = 0;
            TIter nxd(dss);
            while ((k = nxd()) && (fc = (TFileCollection *) dss->GetValue(k))) {
               // Get subset on specified server, if any
               if ((xfc = fc->GetFilesOnServer(server))) {
                  rmap->Add(new TObjString(k->GetName()), xfc);
               }
            }
            dss->DeleteAll();
            delete dss;
            dss = 0;
            if (rmap->GetSize() > 0) {
               dss = rmap;
            } else {
               Printf("GetDataSets: no dataset found on server '%s' for owner '%s'", server, owner);
               delete rmap;
            }
         }
      } else  {
         Printf("GetDataSets: no dataset found for '%s'", owner);
      }
   }
   // Done
   return dss;
}

//_______________________________________________________________________________________
Int_t RemoveDataSet(const char *dsname)
{
   // RemoveDataSet wrapper
   if (gIsProof) {
      doParallel = kFALSE;
      if (!gProof && getProof("RemoveDataSet") != 0) return -1;
      return gProof->RemoveDataSet(dsname);
   } else {
      if (!gDataSetManager && getDSMgr("RemoveDataSet") != 0) return -1;
      return gDataSetManager->RemoveDataSet(dsname);
   }
   // Done
   return -1;
}

//_______________________________________________________________________________________
Int_t VerifyDataSet(const char *dsname, const char *opt, const char *redir)
{
   // VerifyDataSet wrapper

   Int_t rc = -1;
   // Honour the 'redir' if required
   TString srvmaps;
   if (redir && strlen(redir) > 0) srvmaps.Form("|%s", redir);
   if (gIsProof) {
      // Honour the 'redir' if required
      if (!(srvmaps.IsNull())) {
         TProof::AddEnvVar("DATASETSRVMAPS", srvmaps);
      }
      TString sopt(opt);
      doParallel = (sopt.Contains("S")) ? kFALSE : kTRUE;
      if (gProof && doParallel && gProof->GetParallel() == 0) {
         gProof->Close();
         delete gProof;
         gProof = 0;
      }
      if (!gProof && getProof("VerifyDataSet") != 0) return -1;
      if ((rc = gProof->VerifyDataSet(dsname, opt)) == 0) {
         // Success; partial at least. Check if all files are staged
         TFileCollection *fcs = gProof->GetDataSet(dsname, "S:");
         if (fcs && fcs->GetStagedPercentage() < 99.99999) rc = 1;
      }
   } else {
      // Honour the 'redir' if required
      if (!(srvmaps.IsNull())) {
         gEnv->SetValue("DataSet.SrvMaps", srvmaps);
      }
      if (!gDataSetManager && getDSMgr("VerifyDataSet") != 0) return -1;
      if ((rc = gDataSetManager->ScanDataSet(dsname, opt)) == 0) {
         // Success; partial at least. Check if all files are staged
         TFileCollection *fcs = gDataSetManager->GetDataSet(dsname, "S:");
         if (fcs && fcs->GetStagedPercentage() < 99.99999) rc = 1;
      }
   }
   // Done
   return rc;
}

//_______________________________________________________________________________________
Bool_t ExistsDataSet(const char *dsname)
{
   // ExistsDataSet wrapper
   if (gIsProof) {
      doParallel = kFALSE;
      if (!gProof && getProof("ExistsDataSet") != 0) return kFALSE;
      return gProof->ExistsDataSet(dsname);
   } else {
      if (!gDataSetManager && getDSMgr("ExistsDataSet") != 0) return kFALSE;
      return gDataSetManager->ExistsDataSet(dsname);
   }
   return kFALSE;
}

//_______________________________________________________________________________________
Int_t RegisterDataSet(const char *dsname, TFileCollection *fc, const char* opt)
{
   // RegisterDataSet wrapper
   if (gIsProof) {
      doParallel = kFALSE;
      if (!gProof && getProof("GetDataSet") != 0) return -1;
      return gProof->RegisterDataSet(dsname, fc, opt);
   } else {
      if (!gDataSetManager && getDSMgr("RegisterDataSet") != 0) return -1;
      return gDataSetManager->RegisterDataSet(dsname, fc, opt);
   }
   return -1;
}

//_______________________________________________________________________________________
Int_t getProof(const char *where, Int_t verbose)
{
   // Open a PROOF session at gUrl

   {  redirguard rog(flog.Data(), "a", verbose);
      const char *popt = (doParallel) ? "" : "masteronly";
      TProof::Open(gUrl.GetUrl(), popt);
   }
   if (!gProof || !gProof->IsValid()) {
      Printf("getProof:%s: problems starting a PROOF session at '%s'", where, gUrl.GetUrl());
      return -1;
   }
   if (gverbose >= 2) gProof->SetLogLevel(2);
   // Done
   return 0;
}

//_______________________________________________________________________________________
Int_t getDSMgr(const char *where)
{
   // Open a dataset manager for gUrl

   Int_t rc = -1;
   if (gROOT->GetPluginManager()) {
      // Find the appropriate handler
      TPluginHandler *h = gROOT->GetPluginManager()->FindHandler("TDataSetManager", "file");
      if (h && h->LoadPlugin() != -1) {
         TString group(getenv("PQ2GROUP")), user(getenv("PQ2USER"));
         TString dsm, opt("opt:-Ar:-Av:");
         const char *o = getenv("PQ2DSMGROPTS");
         if (o) {
            opt = "";
            if (strlen(o) > 0) opt.Form("opt:%s", o);
         }
         dsm.Form("file dir:%s %s", gUrl.GetUrl(), opt.Data());
         gDataSetManager = reinterpret_cast<TDataSetManager*>(h->ExecPlugin(3,
                                                              group.Data(), user.Data(),
                                                              dsm.Data()));
         if (gDataSetManager) {
            rc = 0;
         } else {
            Printf("getDSMgr:%s: problems creating a dataset manager at '%s'", where, gUrl.GetUrl());
         }
      }
   }
   // Done
   return rc;
}
<|endoftext|>


<commit_before><commit_msg>mapGen.cc file setup<commit_after>
/****************************************************************************************************/
// Global includes

#include <random>

/****************************************************************************************************/
// Local includes

/****************************************************************************************************/
// Using statements

using std::vector;

typedef uchar unsigned char
/****************************************************************************************************/
// Main

int main(int args, char* argv[])
{
    vector<vector<uchar>> world;

    return EXIT_SUCCESS;
}
<|endoftext|>