<commit_before>/*
 * Copyright 2016 neurodata (http://neurodata.io/)
 * Written by Disa Mhembere (disa@jhu.edu)
 *
 * This file is part of knor
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY CURRENT_KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <random>
#include <stdexcept>

#include "gmeans_coordinator.hpp"
#include "gmeans.hpp"

#include "io.hpp"
#include "clusters.hpp"
#include "linalg.hpp"
#include "AndersonDarling.hpp"
#include "hclust_id_generator.hpp"
#include "thd_safe_bool_vector.hpp"

namespace knor {

gmeans_coordinator::gmeans_coordinator(const std::string fn,
        const size_t nrow,
        const size_t ncol, const unsigned k, const unsigned max_iters,
        const unsigned nnodes, const unsigned nthreads,
        const double* centers, const base::init_t it,
        const double tolerance, const base::dist_t dt,
        const unsigned min_clust_size, const short strictness) :
    xmeans_coordinator(fn, nrow, ncol, k, max_iters, nnodes, nthreads,
            centers, it, tolerance, dt, min_clust_size),
            strictness(strictness) {
}

void gmeans_coordinator::deactivate(const unsigned id) {
    cltr_active_vec->check_set(id, false);
}

void gmeans_coordinator::activate(const unsigned id) {
    cltr_active_vec->check_set(id, true);
    if (id > 0)
        curr_nclust++;
}

void gmeans_coordinator::build_thread_state() {
    // NUMA node affinity binding policy is round-robin
    unsigned thds_row = nrow / nthreads;

    for (unsigned thd_id = 0; thd_id < nthreads; thd_id++) {
        std::pair<unsigned, unsigned> tup = get_rid_len_tup(thd_id);
        thd_max_row_idx.push_back((thd_id*thds_row) + tup.second);
        threads.push_back(gmeans::create((thd_id % nnodes),
                    thd_id, tup.first, tup.second,
                    ncol, k, hcltrs, &cluster_assignments[0], fn,
                    _dist_t, cltr_active_vec, partition_dist, nearest_cdist,
                    compute_pdist));
        threads[thd_id]->set_parent_cond(&cond);
        threads[thd_id]->set_parent_pending_threads(&pending_threads);
        threads[thd_id]->start(WAIT); // Thread puts itself to sleep
        std::static_pointer_cast<gmeans>(threads[thd_id])
                    ->set_part_id(&part_id[0]);
        std::static_pointer_cast<gmeans>(threads[thd_id])
                    ->set_g_clusters(cltrs);
    }
}

void gmeans_coordinator::assemble_ad_vecs(std::unordered_map<unsigned,
        std::vector<double>>& ad_vecs) {
    for (size_t i = 0; i < nearest_cdist.size(); i++) {
        ad_vecs[part_id[i]].push_back(nearest_cdist[i]);
    }
}

void gmeans_coordinator::compute_ad_stats(
        std::unordered_map<unsigned, std::vector<double>>& ad_vecs) {

    std::vector<unsigned> keys;
    for (auto const& kv : ad_vecs)
        keys.push_back(kv.first);

    std::vector<double> scores(keys.size());

#pragma omp parallel for
    for (size_t idx = 0; idx < keys.size(); idx++) {
        double score = base::AndersonDarling::compute_statistic(
                ad_vecs[keys[idx]].size(), &(ad_vecs[keys[idx]][0]));
        scores[idx] = score;
    }

    // NOTE: We push the score onto the back
    for (size_t idx = 0; idx < keys.size(); idx++)
        ad_vecs[keys[idx]].push_back(scores[idx]);
}

// NOTE: This modifies hcltrs
void gmeans_coordinator::partition_decision() {
    // Each Anderson Darling (AD) vector represents the vector for which
    //  each cluster gets its AD statistic.
    std::unordered_map<unsigned, std::vector<double>> ad_vecs;
    std::vector<double> critical_values;

    // Populate the AD vectors
    assemble_ad_vecs(ad_vecs);

    for (auto& kv : ad_vecs) {
        base::linalg::scale(&(kv.second)[0], kv.second.size());
    }

    // Compute Critical values
    base::AndersonDarling::compute_critical_values(
            ad_vecs.size(), critical_values);

    // Compute AD statistics
    compute_ad_stats(ad_vecs);

    std::vector<size_t> keys;

    hcltrs.get_keys(keys);
    std::vector<bool> revert_cache;
    auto max_pid = (*std::max_element(keys.begin(), keys.end()));
    revert_cache.assign(max_pid+1, false);

    // NOTE: We use ad_vecs.back() to store the score
    for (size_t i = 0; i < keys.size(); i++) {
        unsigned pid = keys[i];
        auto const& v = ad_vecs[pid];
        auto score = ad_vecs[pid].back();

        if (score <= critical_values[strictness]) {
            unsigned lid = hcltrs[pid]->get_zeroid();
            unsigned rid = hcltrs[pid]->get_oneid();

            // Deactivate both lid and rid
            deactivate(lid); deactivate(rid);
            // Deactivate pid
            deactivate(pid);

            revert_cache[pid] = true;
            hcltrs.erase(pid);
            // We can reuse these children ids
            ider->reclaim_id(lid);
            ider->reclaim_id(rid);
            curr_nclust -= 2;

            final_centroids[pid] = std::vector<double>(
                    cltrs->get_mean_rawptr(pid),
                    cltrs->get_mean_rawptr(pid) + ncol);
            cluster_assignment_counts[pid] =
                cluster_assignment_counts[lid] + cluster_assignment_counts[rid];
            cluster_assignment_counts[lid] = cluster_assignment_counts[rid] = 0;
        }
    }

    // Assemble cluster membership
    // TODO: Use dynamic or guided scheduler
#ifdef _OPENMP
#pragma omp parallel for default(shared) firstprivate(revert_cache)
#endif
    for (size_t rid = 0; rid < nrow; rid++) {
        auto pid = part_id[rid];
        // Ignore keys outside in hcltrs
        if (pid < revert_cache.size() && revert_cache[pid]) {
                // Means that row assignments needs to be reverted to part_id
                cluster_assignments[rid] = pid;
        }
    }
}

void gmeans_coordinator::compute_cluster_diffs() {
    auto itr = hcltrs.get_iterator();
    while (itr.has_next()) {
        auto kv = itr.next();
        auto c = std::static_pointer_cast<base::h_clusters>(kv.second);
        c->metadata.resize(ncol+1); // +1th index stores the divisor

        // Compute difference
        for (size_t i = 0; i < ncol; i++)
            base::linalg::vdiff(c->get_mean_rawptr(0),
                    c->get_mean_rawptr(1), ncol, c->metadata);

        // Compute v.dot(v)
        c->metadata[ncol] = base::linalg::dot(&c->metadata[0],
                &c->metadata[0], ncol); // NOTE: last element intentionally ignored
    }
}

// Main driver
base::cluster_t gmeans_coordinator::run(
        double* allocd_data, const bool numa_opt) {
#ifdef PROFILER
    ProfilerStart("gmeans_coordinator.perf");
#endif

    build_thread_state();

    if (!numa_opt && NULL == allocd_data) {
        wake4run(ALLOC_DATA);
        wait4complete();
    } else if (allocd_data) { // No NUMA opt
        set_thread_data_ptr(allocd_data);
    }

    struct timeval start, end;
    gettimeofday(&start , NULL);

    run_hinit(); // Initialize clusters

    // Run loop
    size_t iter = 0;

    while (true) {
        // TODO: Do this simultaneously with H_EM step
        wake4run(MEAN);
        wait4complete();
        combine_partition_means();
        compute_pdist = true;

        for (iter = 0; iter < max_iters; iter++) {
#ifndef BIND
            printf("\nNCLUST: %lu, Iteration: %lu\n", curr_nclust, iter);
#endif
            // Now pick between the cluster splits
            wake4run(H_EM);
            wait4complete();
            update_clusters();
#ifndef BIND
            printf("\nAssignment counts:\n");
            base::sparse_print(cluster_assignment_counts);
            printf("\n*****************************************************\n");
#endif
            if (compute_pdist)
                compute_pdist = false;
        }

        compute_cluster_diffs();

        wake4run(H_SPLIT);
        wait4complete();

        // Decide on split or not here
        partition_decision();

        if (at_cluster_cap()) {
#ifndef BIND
            printf("\n\nCLUSTER SIZE EXIT @ %lu!\n", curr_nclust);
#endif
            break;
        }

        // Update global state
        init_splits(); // Initialize possible splits

        // Break when clusters are inactive due to size
        if (hcltrs.keyless()) {
            assert(steady_state()); // NOTE: Comment when benchmarking
#ifndef BIND
            printf("\n\nSTEADY STATE EXIT!\n");
#endif
            break;
        }
    }
    complete_final_centroids();

#ifdef PROFILER
    ProfilerStop();
#endif

    gettimeofday(&end, NULL);
#ifndef BIND
    printf("\n\nAlgorithmic time taken = %.6f sec\n",
        base::time_diff(start, end));
    printf("\n******************************************\n");
    verify_consistency();
#endif

#ifndef BIND
    printf("Final cluster counts: \n");
    base::sparse_print(cluster_assignment_counts);
    //printf("Final centroids\n");
    //for (auto const& kv : final_centroids) {
        //printf("k: %u, v: ", kv.first); base::print(kv.second);
    //}
    printf("\n******************************************\n");
#endif

    return base::cluster_t(this->nrow, this->ncol, iter,
            cluster_assignments, cluster_assignment_counts,
            final_centroids);
}
} // End namespace knor

<commit_msg>remove unused<commit_after>/*
 * Copyright 2016 neurodata (http://neurodata.io/)
 * Written by Disa Mhembere (disa@jhu.edu)
 *
 * This file is part of knor
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY CURRENT_KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <random>
#include <stdexcept>

#include "gmeans_coordinator.hpp"
#include "gmeans.hpp"

#include "io.hpp"
#include "clusters.hpp"
#include "linalg.hpp"
#include "AndersonDarling.hpp"
#include "hclust_id_generator.hpp"
#include "thd_safe_bool_vector.hpp"

namespace knor {

gmeans_coordinator::gmeans_coordinator(const std::string fn,
        const size_t nrow,
        const size_t ncol, const unsigned k, const unsigned max_iters,
        const unsigned nnodes, const unsigned nthreads,
        const double* centers, const base::init_t it,
        const double tolerance, const base::dist_t dt,
        const unsigned min_clust_size, const short strictness) :
    xmeans_coordinator(fn, nrow, ncol, k, max_iters, nnodes, nthreads,
            centers, it, tolerance, dt, min_clust_size),
            strictness(strictness) {
}

void gmeans_coordinator::deactivate(const unsigned id) {
    cltr_active_vec->check_set(id, false);
}

void gmeans_coordinator::activate(const unsigned id) {
    cltr_active_vec->check_set(id, true);
    if (id > 0)
        curr_nclust++;
}

void gmeans_coordinator::build_thread_state() {
    // NUMA node affinity binding policy is round-robin
    unsigned thds_row = nrow / nthreads;

    for (unsigned thd_id = 0; thd_id < nthreads; thd_id++) {
        std::pair<unsigned, unsigned> tup = get_rid_len_tup(thd_id);
        thd_max_row_idx.push_back((thd_id*thds_row) + tup.second);
        threads.push_back(gmeans::create((thd_id % nnodes),
                    thd_id, tup.first, tup.second,
                    ncol, k, hcltrs, &cluster_assignments[0], fn,
                    _dist_t, cltr_active_vec, partition_dist, nearest_cdist,
                    compute_pdist));
        threads[thd_id]->set_parent_cond(&cond);
        threads[thd_id]->set_parent_pending_threads(&pending_threads);
        threads[thd_id]->start(WAIT); // Thread puts itself to sleep
        std::static_pointer_cast<gmeans>(threads[thd_id])
                    ->set_part_id(&part_id[0]);
        std::static_pointer_cast<gmeans>(threads[thd_id])
                    ->set_g_clusters(cltrs);
    }
}

void gmeans_coordinator::assemble_ad_vecs(std::unordered_map<unsigned,
        std::vector<double>>& ad_vecs) {
    for (size_t i = 0; i < nearest_cdist.size(); i++) {
        ad_vecs[part_id[i]].push_back(nearest_cdist[i]);
    }
}

void gmeans_coordinator::compute_ad_stats(
        std::unordered_map<unsigned, std::vector<double>>& ad_vecs) {

    std::vector<unsigned> keys;
    for (auto const& kv : ad_vecs)
        keys.push_back(kv.first);

    std::vector<double> scores(keys.size());

#pragma omp parallel for
    for (size_t idx = 0; idx < keys.size(); idx++) {
        double score = base::AndersonDarling::compute_statistic(
                ad_vecs[keys[idx]].size(), &(ad_vecs[keys[idx]][0]));
        scores[idx] = score;
    }

    // NOTE: We push the score onto the back
    for (size_t idx = 0; idx < keys.size(); idx++)
        ad_vecs[keys[idx]].push_back(scores[idx]);
}

// NOTE: This modifies hcltrs
void gmeans_coordinator::partition_decision() {
    // Each Anderson Darling (AD) vector represents the vector for which
    //  each cluster gets its AD statistic.
    std::unordered_map<unsigned, std::vector<double>> ad_vecs;
    std::vector<double> critical_values;

    // Populate the AD vectors
    assemble_ad_vecs(ad_vecs);

    for (auto& kv : ad_vecs) {
        base::linalg::scale(&(kv.second)[0], kv.second.size());
    }

    // Compute Critical values
    base::AndersonDarling::compute_critical_values(
            ad_vecs.size(), critical_values);

    // Compute AD statistics
    compute_ad_stats(ad_vecs);

    std::vector<size_t> keys;

    hcltrs.get_keys(keys);
    std::vector<bool> revert_cache;
    auto max_pid = (*std::max_element(keys.begin(), keys.end()));
    revert_cache.assign(max_pid+1, false);

    // NOTE: We use ad_vecs.back() to store the score
    for (size_t i = 0; i < keys.size(); i++) {
        unsigned pid = keys[i];
        auto score = ad_vecs[pid].back();

        if (score <= critical_values[strictness]) {
            unsigned lid = hcltrs[pid]->get_zeroid();
            unsigned rid = hcltrs[pid]->get_oneid();

            // Deactivate both lid and rid
            deactivate(lid); deactivate(rid);
            // Deactivate pid
            deactivate(pid);

            revert_cache[pid] = true;
            hcltrs.erase(pid);
            // We can reuse these children ids
            ider->reclaim_id(lid);
            ider->reclaim_id(rid);
            curr_nclust -= 2;

            final_centroids[pid] = std::vector<double>(
                    cltrs->get_mean_rawptr(pid),
                    cltrs->get_mean_rawptr(pid) + ncol);
            cluster_assignment_counts[pid] =
                cluster_assignment_counts[lid] + cluster_assignment_counts[rid];
            cluster_assignment_counts[lid] = cluster_assignment_counts[rid] = 0;
        }
    }

    // Assemble cluster membership
    // TODO: Use dynamic or guided scheduler
#ifdef _OPENMP
#pragma omp parallel for default(shared) firstprivate(revert_cache)
#endif
    for (size_t rid = 0; rid < nrow; rid++) {
        auto pid = part_id[rid];
        // Ignore keys outside in hcltrs
        if (pid < revert_cache.size() && revert_cache[pid]) {
                // Means that row assignments needs to be reverted to part_id
                cluster_assignments[rid] = pid;
        }
    }
}

void gmeans_coordinator::compute_cluster_diffs() {
    auto itr = hcltrs.get_iterator();
    while (itr.has_next()) {
        auto kv = itr.next();
        auto c = std::static_pointer_cast<base::h_clusters>(kv.second);
        c->metadata.resize(ncol+1); // +1th index stores the divisor

        // Compute difference
        for (size_t i = 0; i < ncol; i++)
            base::linalg::vdiff(c->get_mean_rawptr(0),
                    c->get_mean_rawptr(1), ncol, c->metadata);

        // Compute v.dot(v)
        c->metadata[ncol] = base::linalg::dot(&c->metadata[0],
                &c->metadata[0], ncol); // NOTE: last element intentionally ignored
    }
}

// Main driver
base::cluster_t gmeans_coordinator::run(
        double* allocd_data, const bool numa_opt) {
#ifdef PROFILER
    ProfilerStart("gmeans_coordinator.perf");
#endif

    build_thread_state();

    if (!numa_opt && NULL == allocd_data) {
        wake4run(ALLOC_DATA);
        wait4complete();
    } else if (allocd_data) { // No NUMA opt
        set_thread_data_ptr(allocd_data);
    }

    struct timeval start, end;
    gettimeofday(&start , NULL);

    run_hinit(); // Initialize clusters

    // Run loop
    size_t iter = 0;

    while (true) {
        // TODO: Do this simultaneously with H_EM step
        wake4run(MEAN);
        wait4complete();
        combine_partition_means();
        compute_pdist = true;

        for (iter = 0; iter < max_iters; iter++) {
#ifndef BIND
            printf("\nNCLUST: %lu, Iteration: %lu\n", curr_nclust, iter);
#endif
            // Now pick between the cluster splits
            wake4run(H_EM);
            wait4complete();
            update_clusters();
#ifndef BIND
            printf("\nAssignment counts:\n");
            base::sparse_print(cluster_assignment_counts);
            printf("\n*****************************************************\n");
#endif
            if (compute_pdist)
                compute_pdist = false;
        }

        compute_cluster_diffs();

        wake4run(H_SPLIT);
        wait4complete();

        // Decide on split or not here
        partition_decision();

        if (at_cluster_cap()) {
#ifndef BIND
            printf("\n\nCLUSTER SIZE EXIT @ %lu!\n", curr_nclust);
#endif
            break;
        }

        // Update global state
        init_splits(); // Initialize possible splits

        // Break when clusters are inactive due to size
        if (hcltrs.keyless()) {
            assert(steady_state()); // NOTE: Comment when benchmarking
#ifndef BIND
            printf("\n\nSTEADY STATE EXIT!\n");
#endif
            break;
        }
    }
    complete_final_centroids();

#ifdef PROFILER
    ProfilerStop();
#endif

    gettimeofday(&end, NULL);
#ifndef BIND
    printf("\n\nAlgorithmic time taken = %.6f sec\n",
        base::time_diff(start, end));
    printf("\n******************************************\n");
    verify_consistency();
#endif

#ifndef BIND
    printf("Final cluster counts: \n");
    base::sparse_print(cluster_assignment_counts);
    //printf("Final centroids\n");
    //for (auto const& kv : final_centroids) {
        //printf("k: %u, v: ", kv.first); base::print(kv.second);
    //}
    printf("\n******************************************\n");
#endif

    return base::cluster_t(this->nrow, this->ncol, iter,
            cluster_assignments, cluster_assignment_counts,
            final_centroids);
}
} // End namespace knor

<|endoftext|>


<commit_before>#include <unit_test_merge_text.h>
#include <utils_merge_text.h>
#include <vector>
#include <iostream>

REGISTRY_UNIT_TEST(UnitTestMergeText)

UnitTestMergeText::UnitTestMergeText()
    : UnitTestBase("UnitTestMergeText") {
    //
}

// ---------------------------------------------------------------------

void UnitTestMergeText::init() {
}

// ---------------------------------------------------------------------

bool UnitTestMergeText::run() {

    struct LTest {
        LTest(
            std::string txt1,
            std::string txt2,
            std::string txt3,
            std::vector<UtilsMergeTextRow *> arr1,
            std::vector<UtilsMergeTextRow *> arr2,
            unsigned int n
        ) : txt1(txt1), txt2(txt2), txt3(txt3), arr1(arr1), arr2(arr2), n(n) {
            //
        }
        std::string txt1;
        std::string txt2;
        std::string txt3;
        std::vector<UtilsMergeTextRow *> arr1;
        std::vector<UtilsMergeTextRow *> arr2;
        unsigned int n;
    };

    std::vector<LTest *> tests1;
    std::vector<UtilsMergeTextRow *> arr1, arr2;

    std::string text1 = text001();
    std::string text2 = text002();

    // std::cout << text1.toStdString() << "\n";

    //empty (Test №1)
    tests1.push_back(new LTest("", "", "", arr1, arr2, 0));
    //without any difference (Test №2)
    tests1.push_back(new LTest("I\r\ngot\r\nMike", "I\r\ngot\r\nMike", "I\r\ngot\r\nMike", arr1, arr2, 0));
    //with a difference in the ending (Tests №3-6)
    tests1.push_back(new LTest("I/got/Mike", "I/got/Mike", "I/got/Nike", arr1, arr2, 1));
    tests1.push_back(new LTest("I/got/Nike", "I/got/Mike", "I/got/Mike", arr1, arr2, 1));
    tests1.push_back(new LTest("I/got/Nike", "I/got/Mike", "I/got/Nike", arr1, arr2, 1));
    tests1.push_back(new LTest("I/got/Bike", "I/got/Mike", "I/got/Nike", arr1, arr2, 1));
    //with a difference in the beginning (Tests №7-10)
    tests1.push_back(new LTest("I/got/Mike", "I/got/Mike", "You/got/Mike", arr1, arr2, 1));
    tests1.push_back(new LTest("You/got/Mike", "I/got/Mike", "I/got/Mike", arr1, arr2, 1));
    tests1.push_back(new LTest("You/got/Mike", "I/got/Mike", "You/got/Mike", arr1, arr2, 1));
    tests1.push_back(new LTest("We/got/Mike", "I/got/Mike", "You/got/Mike", arr1, arr2, 1));
    //with a difference in the middle of the row (Tests №11-15)
    tests1.push_back(new LTest("I/have/cute/cats", "I/have/cute/cats", "I/have/huge/cats", arr1, arr2, 1));
    tests1.push_back(new LTest("I/have/huge/cats", "I/have/cute/cats", "I/have/cute/cats", arr1, arr2, 1));
    tests1.push_back(new LTest("I/have/huge/cats", "I/have/cute/cats", "I/have/huge/cats", arr1, arr2, 1));
    tests1.push_back(new LTest("I/have/difference/cats", "I/have/cute/cats", "I/have/huge/cats", arr1, arr2, 1));
    tests1.push_back(new LTest("I/betray/cats", "I/have/cats", "I/love/cats", arr1, arr2, 1));
    //delete lines from the end of text (Tests №16-19)
    tests1.push_back(new LTest("Go\r\nto\r\nthe\r\nschool", "Go\r\nto\r\nthe\r\nschool", "Go\r\nto", arr1, arr2, 2));
    tests1.push_back(new LTest("Go/to", "Go/to/the/school", "Go/to/the/school", arr1, arr2, 2));
    tests1.push_back(new LTest("Go/to/the", "Go/to/the/school", "Go/to", arr1, arr2, 2));
    tests1.push_back(new LTest("Go/to", "Go/to/the/school", "Go/to/the", arr1, arr2, 2));
    //add lines in the end of text (Tests №20-23)
    tests1.push_back(new LTest("Go/to", "Go/to", "Go/to/the/school", arr1, arr2, 2));
    tests1.push_back(new LTest("Go/to/the/school", "Go/to", "Go/to", arr1, arr2, 2));
    tests1.push_back(new LTest("Go/to/the", "Go/to", "Go/to/the/school", arr1, arr2, 2));
    tests1.push_back(new LTest("Go/to/the/school", "Go/to", "Go/to/the", arr1, arr2, 2));
    //delete rows from the beginning and the middle of text (Tests №24-27)
    tests1.push_back(new LTest("Go/to/the/school", "Go/to/the/school", "the/school", arr1, arr2, 2));
    tests1.push_back(new LTest("the/school", "Go/to/the/school", "Go/to/the/school", arr1, arr2, 2));
    tests1.push_back(new LTest("Go/to/the/school", "Go/to/the/school", "Go/to/school", arr1, arr2, 1));
    tests1.push_back(new LTest("Go/to/school", "Go/to/the/school", "Go/to/the/school", arr1, arr2, 1));
    //add rows to the beginning and the middle of text (Tests №28-31)
    tests1.push_back(new LTest("the/school", "the/school", "Go/to/the/school", arr1, arr2, 2));
    tests1.push_back(new LTest("Go/to/the/school", "the/school", "the/school", arr1, arr2, 2));
    tests1.push_back(new LTest("Go/the/school", "the/school", "the/school", arr1, arr2, 1));
    tests1.push_back(new LTest("the/school", "the/school", "Go/the/school", arr1, arr2, 1));
    //complex (Tests №32-33)
    tests1.push_back(new LTest("You/are/got/and/good", "I/got/Mike/and/this is/good/boy", "I/cure/damn/ill/cancer", arr1, arr2, 8));
    tests1.push_back(new LTest("We/spent/the rest/of our life/on/this/course/work", "We/spent/a lot of/time/on/this/course/work", "We/gonna/spend/a lot of/funky/time/on/course/work", arr1, arr2, 6));

    unsigned int nSuccess = 0;
    for (unsigned int i = 0; i < tests1.size(); i++){
        std::string txt1 = tests1[i]->txt1;
        std::string txt2 = tests1[i]->txt2;
        std::string txt3 = tests1[i]->txt3;
        UtilsMergeText::merge(txt1, txt2, txt3, arr1, arr2);
        unsigned int n = tests1[i]->n;

        if(arr1.size()==n) {
            nSuccess++;
        } else {
            Log::err(TAG, "In the test №" + std::to_string(i + 1) + " the length of the vector is expected: "
                 + std::to_string(n) + ", but obtained: " + std::to_string(arr1.size()));
        }
        arr1.clear(), arr2.clear();
    }

    //sort module test
    struct LTest2 {
        LTest2(
            std::string txt1,
            std::string txt2,
            std::string txt3,
            std::vector<UtilsMergeTextRow *> arr1,
            std::vector<UtilsMergeTextRow *> arr2,
            std::vector<UtilsMergeTextRow *> arr3
        ) : txt1(txt1), txt2(txt2), txt3(txt3), arr1(arr1), arr2(arr2), arr3(arr3) {
            //
        }

        std::string txt1;
        std::string txt2;
        std::string txt3;
        std::vector<UtilsMergeTextRow *> arr1;
        std::vector<UtilsMergeTextRow *> arr2;
        std::vector<UtilsMergeTextRow *> arr3;
    };

    std::vector<LTest2 *> tests2;
    std::vector<UtilsMergeTextRow *> arr3;
    int id[8] = {0,1,1,2,3,4,5,6};
    std::vector<std::string> skey;
    skey.push_back("I");
    skey.push_back("got");
    skey.push_back("!add");
    skey.push_back("Mike");
    skey.push_back("and");
    skey.push_back("this is");
    skey.push_back("!del");
    skey.push_back("!del");

    std::vector<std::string> sline;
    sline.push_back("You");
    sline.push_back("cure");
    sline.push_back("are");
    sline.push_back("damn");
    sline.push_back("ill");
    sline.push_back("cancer");
    sline.push_back("good");
    sline.push_back("boy");

    for(int i=0;i<skey.size();++i) arr3.push_back(new UtilsMergeTextRow(id[i], skey.at(i), sline.at(i)));

    tests2.push_back(new LTest2(
        "You\r\nare\r\ngot\r\nand\r\ngood",
        "I\r\ngot\r\nMike\r\nand\r\nthis is\r\ngood\r\nboy",
        "I\r\ncure\r\ndamn\r\nill\r\ncancer",
        arr1, arr2, arr3));

    std::string txt1 = tests2[0]->txt1;
    std::string txt2 = tests2[0]->txt2;
    std::string txt3 = tests2[0]->txt3;
    UtilsMergeText::merge(txt1, txt2, txt3, arr1, arr2);

    unsigned int Success = 0;
    for (int i = 0; i < 8; ++i) {
        int id1 = arr1.at(i)->id;
        int id2 = arr3.at(i)->id;
        std::string key1 = arr1.at(i)->key;
        std::string key2 = arr3.at(i)->key;
        std::string line1 = arr1.at(i)->line;
        std::string line2 = arr3.at(i)->line;
        if (id1==id2 && key1==key2 && line1==line2) {
            Success+=1;
        } else {
            Log::info(TAG, "In the sort test in the element №" + std::to_string(i+1) + ":");

            if (id1!=id2) {
                Log::err(TAG, "expected id: '" + std::to_string(id2) + "', but obtained: '" + std::to_string(id1) + "'");
            }

            if (key1!=key2) {
                Log::err(TAG, "expected key: '" + key2 + "', but obtained: '" + key1 + "'");
            }

            if (line1!=line2) {
                Log::err(TAG, "expected line: '" + line2 + "', but obtained: '" + line1 + "'");
            }
        }
    }

    if (Success == 8) {
        nSuccess++;
    }

    return nSuccess == tests1.size() + 1;
}

// ---------------------------------------------------------------------

std::string UnitTestMergeText::text001() {
    return "Str1\r\n"
        "Str\r\n"
        "Str\r\n"
        "Str\r\n"
        "Str2\r\n"
        "Str3\r\n"
        "Str\r\n"
        "Str\r\n"
        "Str\r\n";
}

// ---------------------------------------------------------------------

std::string UnitTestMergeText::text002() {
    return "Str1\r\n"
        "Str \r\n"
        "Str\r\n"
        "Str\r\n"
        "Str2\r\n"
        "Str\r\n"
        "Str3\r\n"
        "Str\r\n"
        "Str\r\n"
        "Str\r\n";
}

// ---------------------------------------------------------------------<commit_msg>Fixed unit for merge<commit_after>#include <unit_test_merge_text.h>
#include <utils_merge_text.h>
#include <vector>
#include <iostream>

REGISTRY_UNIT_TEST(UnitTestMergeText)

UnitTestMergeText::UnitTestMergeText()
    : UnitTestBase("UnitTestMergeText") {
    //
}

// ---------------------------------------------------------------------

void UnitTestMergeText::init() {
}

// ---------------------------------------------------------------------

bool UnitTestMergeText::run() {

    struct LTest {
        LTest(
            std::string txt1,
            std::string txt2,
            std::string txt3,
            std::vector<UtilsMergeTextRow *> arr1,
            std::vector<UtilsMergeTextRow *> arr2,
            unsigned int n
        ) : txt1(txt1), txt2(txt2), txt3(txt3), arr1(arr1), arr2(arr2), n(n) {
            //
        }
        std::string txt1;
        std::string txt2;
        std::string txt3;
        std::vector<UtilsMergeTextRow *> arr1;
        std::vector<UtilsMergeTextRow *> arr2;
        unsigned int n;
    };

    std::vector<LTest *> tests1;
    std::vector<UtilsMergeTextRow *> arr1, arr2;

    std::string text1 = text001();
    std::string text2 = text002();

    // std::cout << text1.toStdString() << "\n";

    //empty (Test №1)
    tests1.push_back(new LTest("", "", "", arr1, arr2, 0));
    //without any difference (Test №2)
    tests1.push_back(new LTest("I\ngot\nMike", "I\ngot\nMike", "I\ngot\nMike", arr1, arr2, 0));
    //with a difference in the ending (Tests №3-6)
    tests1.push_back(new LTest("I\ngot\nMike", "I\ngot\nMike", "I\ngot\nNike", arr1, arr2, 1));
    tests1.push_back(new LTest("I\ngot\nNike", "I\ngot\nMike", "I\ngot\nMike", arr1, arr2, 1));
    tests1.push_back(new LTest("I\ngot\nNike", "I\ngot\nMike", "I\ngot\nNike", arr1, arr2, 1));
    tests1.push_back(new LTest("I\ngot\nBike", "I\ngot\nMike", "I\ngot\nNike", arr1, arr2, 1));
    //with a difference in the beginning (Tests №7-10)
    tests1.push_back(new LTest("I\ngot\nMike", "I\ngot\nMike", "You\ngot\nMike", arr1, arr2, 1));
    tests1.push_back(new LTest("You\ngot\nMike", "I\ngot\nMike", "I\ngot\nMike", arr1, arr2, 1));
    tests1.push_back(new LTest("You\ngot\nMike", "I\ngot\nMike", "You\ngot\nMike", arr1, arr2, 1));
    tests1.push_back(new LTest("We\ngot\nMike", "I\ngot\nMike", "You\ngot\nMike", arr1, arr2, 1));
    //with a difference in the middle of the row (Tests №11-15)
    tests1.push_back(new LTest("I\nhave\ncute\ncats", "I\nhave\ncute\ncats", "I\nhave\nhuge\ncats", arr1, arr2, 1));
    tests1.push_back(new LTest("I\nhave\nhuge\ncats", "I\nhave\ncute\ncats", "I\nhave\ncute\ncats", arr1, arr2, 1));
    tests1.push_back(new LTest("I\nhave\nhuge\ncats", "I\nhave\ncute\ncats", "I\nhave\nhuge\ncats", arr1, arr2, 1));
    tests1.push_back(new LTest("I\nhave\ndifference\ncats", "I\nhave\ncute\ncats", "I\nhave\nhuge\ncats", arr1, arr2, 1));
    tests1.push_back(new LTest("I\nbetray\ncats", "I\nhave\ncats", "I\nlove\ncats", arr1, arr2, 1));
    //delete lines from the end of text (Tests №16-19)
    tests1.push_back(new LTest("Go\nto\nthe\nschool", "Go\nto\nthe\nschool", "Go\nto", arr1, arr2, 2));
    tests1.push_back(new LTest("Go\nto", "Go\nto\nthe\nschool", "Go\nto\nthe\nschool", arr1, arr2, 2));
    tests1.push_back(new LTest("Go\nto\nthe", "Go\nto\nthe\nschool", "Go\nto", arr1, arr2, 2));
    tests1.push_back(new LTest("Go\nto", "Go\nto\nthe\nschool", "Go\nto\nthe", arr1, arr2, 2));
    //add lines in the end of text (Tests №20-23)
    tests1.push_back(new LTest("Go\nto", "Go\nto", "Go\nto\nthe\nschool", arr1, arr2, 2));
    tests1.push_back(new LTest("Go\nto\nthe\nschool", "Go\nto", "Go\nto", arr1, arr2, 2));
    tests1.push_back(new LTest("Go\nto\nthe", "Go\nto", "Go\nto\nthe\nschool", arr1, arr2, 2));
    tests1.push_back(new LTest("Go\nto\nthe\nschool", "Go\nto", "Go\nto\nthe", arr1, arr2, 2));
    //delete rows from the beginning and the middle of text (Tests №24-27)
    tests1.push_back(new LTest("Go\nto\nthe\nschool", "Go\nto\nthe\nschool", "the\nschool", arr1, arr2, 2));
    tests1.push_back(new LTest("the\nschool", "Go\nto\nthe\nschool", "Go\nto\nthe\nschool", arr1, arr2, 2));
    tests1.push_back(new LTest("Go\nto\nthe\nschool", "Go\nto\nthe\nschool", "Go\nto\nschool", arr1, arr2, 1));
    tests1.push_back(new LTest("Go\nto\nschool", "Go\nto\nthe\nschool", "Go\nto\nthe\nschool", arr1, arr2, 1));
    //add rows to the beginning and the middle of text (Tests №28-31)
    tests1.push_back(new LTest("the\nschool", "the\nschool", "Go\nto\nthe\nschool", arr1, arr2, 2));
    tests1.push_back(new LTest("Go\nto\nthe\nschool", "the\nschool", "the\nschool", arr1, arr2, 2));
    tests1.push_back(new LTest("Go\nthe\nschool", "the\nschool", "the\nschool", arr1, arr2, 1));
    tests1.push_back(new LTest("the\nschool", "the\nschool", "Go\nthe\nschool", arr1, arr2, 1));
    //complex (Tests №32-33)
    tests1.push_back(new LTest("You\nare\ngot\nand\ngood", "I\ngot\nMike\nand\nthis is\ngood\nboy", "I\ncure\ndamn\nill\ncancer", arr1, arr2, 8));
    tests1.push_back(new LTest("We\nspent\nthe rest\nof our life\non\nthis\ncourse\nwork", "We\nspent\na lot of\ntime\non\nthis\ncourse\nwork", "We\ngonna\nspend\na lot of\nfunky\ntime\non\ncourse\nwork", arr1, arr2, 6));

    unsigned int nSuccess = 0;
    for (unsigned int i = 0; i < tests1.size(); i++){
        std::string txt1 = tests1[i]->txt1;
        std::string txt2 = tests1[i]->txt2;
        std::string txt3 = tests1[i]->txt3;
        UtilsMergeText::merge(txt1, txt2, txt3, arr1, arr2);
        unsigned int n = tests1[i]->n;

        if(arr1.size()==n) {
            nSuccess++;
        } else {
            Log::err(TAG, "In the test №" + std::to_string(i + 1) + " the length of the vector is expected: "
                 + std::to_string(n) + ", but obtained: " + std::to_string(arr1.size()));
        }
        arr1.clear(), arr2.clear();
    }

    //sort module test
    struct LTest2 {
        LTest2(
            std::string txt1,
            std::string txt2,
            std::string txt3,
            std::vector<UtilsMergeTextRow *> arr1,
            std::vector<UtilsMergeTextRow *> arr2,
            std::vector<UtilsMergeTextRow *> arr3
        ) : txt1(txt1), txt2(txt2), txt3(txt3), arr1(arr1), arr2(arr2), arr3(arr3) {
            //
        }

        std::string txt1;
        std::string txt2;
        std::string txt3;
        std::vector<UtilsMergeTextRow *> arr1;
        std::vector<UtilsMergeTextRow *> arr2;
        std::vector<UtilsMergeTextRow *> arr3;
    };

    std::vector<LTest2 *> tests2;
    std::vector<UtilsMergeTextRow *> arr3;
    int id[8] = {0,1,1,2,3,4,5,6};
    std::vector<std::string> skey;
    skey.push_back("I");
    skey.push_back("got");
    skey.push_back("!add");
    skey.push_back("Mike");
    skey.push_back("and");
    skey.push_back("this is");
    skey.push_back("!del");
    skey.push_back("!del");

    std::vector<std::string> sline;
    sline.push_back("You");
    sline.push_back("cure");
    sline.push_back("are");
    sline.push_back("damn");
    sline.push_back("ill");
    sline.push_back("cancer");
    sline.push_back("good");
    sline.push_back("boy");

    for(int i=0;i<skey.size();++i) arr3.push_back(new UtilsMergeTextRow(id[i], skey.at(i), sline.at(i)));

    tests2.push_back(new LTest2(
        "You\nare\ngot\nand\ngood",
        "I\ngot\nMike\nand\nthis is\ngood\nboy",
        "I\ncure\ndamn\nill\ncancer",
        arr1, arr2, arr3));

    std::string txt1 = tests2[0]->txt1;
    std::string txt2 = tests2[0]->txt2;
    std::string txt3 = tests2[0]->txt3;
    UtilsMergeText::merge(txt1, txt2, txt3, arr1, arr2);

    unsigned int Success = 0;
    for (int i = 0; i < 8; ++i) {
        int id1 = arr1.at(i)->id;
        int id2 = arr3.at(i)->id;
        std::string key1 = arr1.at(i)->key;
        std::string key2 = arr3.at(i)->key;
        std::string line1 = arr1.at(i)->line;
        std::string line2 = arr3.at(i)->line;
        if (id1==id2 && key1==key2 && line1==line2) {
            Success+=1;
        } else {
            Log::info(TAG, "In the sort test in the element №" + std::to_string(i+1) + ":");

            if (id1!=id2) {
                Log::err(TAG, "expected id: '" + std::to_string(id2) + "', but obtained: '" + std::to_string(id1) + "'");
            }

            if (key1!=key2) {
                Log::err(TAG, "expected key: '" + key2 + "', but obtained: '" + key1 + "'");
            }

            if (line1!=line2) {
                Log::err(TAG, "expected line: '" + line2 + "', but obtained: '" + line1 + "'");
            }
        }
    }

    if (Success == 8) {
        nSuccess++;
    }

    return nSuccess == tests1.size() + 1;
}

// ---------------------------------------------------------------------

std::string UnitTestMergeText::text001() {
    return "Str1\n"
        "Str\n"
        "Str\n"
        "Str\n"
        "Str2\n"
        "Str3\n"
        "Str\n"
        "Str\n"
        "Str\n";
}

// ---------------------------------------------------------------------

std::string UnitTestMergeText::text002() {
    return "Str1\n"
        "Str \n"
        "Str\n"
        "Str\n"
        "Str2\n"
        "Str\n"
        "Str3\n"
        "Str\n"
        "Str\n"
        "Str\n";
}

// ---------------------------------------------------------------------
<|endoftext|>


<commit_before>/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/common_runtime/control_flow_deps_to_chains.h"
#include <cstdint>

#include "tensorflow/core/framework/attr_value.pb.h"
#include "tensorflow/core/framework/node_def.pb.h"
#include "tensorflow/core/framework/node_def_util.h"
#include "tensorflow/core/framework/op_def_builder.h"
#include "tensorflow/core/framework/tensor.pb.h"
#include "tensorflow/core/platform/errors.h"
#include "tensorflow/core/platform/strcat.h"
#include "tensorflow/core/util/dump_graph.h"

namespace tensorflow {

// TODO(mdan): Move this into Grappler - cleaner interface.
Status ControlFlowDepsToChainsPass::Run(
    const GraphOptimizationPassOptions& options) {
  VLOG(1) << "ControlFlowDepsToChainsPass::Run";

  if (options.graph == nullptr) {
    VLOG(1) << "ControlFlowDepsToChainsPass::Run Aborted";
    return Status::OK();
  }

  Graph* g = options.graph->get();
  DCHECK(g != nullptr);
  FunctionLibraryDefinition* flib_def = options.flib_def;
  DCHECK(flib_def != nullptr);

  if (VLOG_IS_ON(1)) {
    DumpGraphToFile("control_flow_deps_to_chains_before", *g, flib_def);
  }

  for (Node* n : g->nodes()) {
    if (n == nullptr) continue;
    if (!n->IsWhileNode()) continue;

    // TODO(mdan): This breaks encapsulation of Node/Graph. Is there any needed?
    // TODO(mdan): Consolidate this with AddWhileInputHack.
    NodeDef* while_node = n->mutable_def();
    const auto& attrs = while_node->attr();
    auto* mattrs = while_node->mutable_attr();

    string body_name = attrs.at("body").func().name();
    auto* body_graph = flib_def->Find(body_name);
    DCHECK(body_graph != nullptr);

    // Look for required annotations.

    if (attrs.find("_stateful_parallelism") == attrs.end()) continue;
    if (!attrs.at("_stateful_parallelism").b()) continue;
    // TODO(mdan): We don't really need this attribute.
    if (attrs.find("_num_original_outputs") == attrs.end()) continue;
    int body_barrier_loc = -1;
    std::map<string, int> node_index;
    for (int i = 0, s = body_graph->node_def_size(); i < s; i++) {
      node_index.emplace(body_graph->node_def(i).name(), i);
      if (body_barrier_loc < 0) {
        const auto& node_attr = body_graph->node_def(i).attr();
        if (node_attr.find("_acd_function_control_output") != node_attr.end()) {
          body_barrier_loc = i;
        }
      }
    }
    if (body_barrier_loc < 0) continue;
    bool ok_for_lowering = true;
    for (int i = 0; i < body_graph->control_ret_size(); i++) {
      const auto& control_node = body_graph->node_def(
          node_index[body_graph->signature().control_output(i)]);
      const auto& control_attr = control_node.attr();
      if (control_attr.find("_res_first_used_by") == control_attr.end()) {
        ok_for_lowering = false;
        break;
      }
    }
    if (!ok_for_lowering) continue;

    int num_loop_vars = body_graph->signature().input_arg_size();
    int num_new_chains = body_graph->control_ret_size();
    int num_node_inputs = while_node->input_size();

    if (!num_new_chains) continue;  // Nothing to do for stateless loops.

    // Add extra loop vars to the while node.

    // TODO(mdan): If the loop vars contains the resource, we should reuse it.
    // Note that stateful ops of resource inputs cause their resources to be
    // captured into the loop vars (through the body/cond captures). We could
    // effectively use those as chains.

    // TODO(mdan): Is there a more efficient way to do this?
    // Insert the new While node inputs: at the end of the loop vars, but before
    // any non-loop var inputs (like control dependencies). Once the initial
    // chain values are created below, they will be added to these inputs.
    for (int i = 0; i < num_new_chains; i++) {
      while_node->add_input();
    }
    for (int i = num_node_inputs - 1; i >= num_loop_vars; i--) {
      while_node->set_input(i + num_new_chains, while_node->input(i));
    }

    std::vector<Node*> new_inputs;
    std::vector<int> new_input_locations;
    // Set their name to a gensym, type to float and shape to scalar.
    for (int i = 0; i < num_new_chains; i++) {
      string c_name = g->NewName("acd__chain");

      // The initial value for the i'th chain loop var.
      NodeDef new_in;
      new_in.set_name(c_name);
      new_in.set_op("Const");
      AttrValue att_dtype;
      att_dtype.set_type(DT_FLOAT);
      new_in.mutable_attr()->insert({"dtype", att_dtype});
      AttrValue att_value;
      att_value.mutable_tensor()->set_dtype(DT_FLOAT);
      att_value.mutable_tensor()->mutable_tensor_shape();
      att_value.mutable_tensor()->add_int_val(0);
      new_in.mutable_attr()->insert({"value", att_value});
      Status status;
      new_inputs.push_back(g->AddNode(new_in, &status));
      TF_RETURN_WITH_CONTEXT_IF_ERROR(status, "while creating chain", c_name);

      int loc = num_loop_vars + i;
      new_input_locations.push_back(loc);
      while_node->set_input(loc, c_name);
      mattrs->at("T").mutable_list()->add_type(DT_FLOAT);
      mattrs->at("output_shapes").mutable_list()->add_shape();
    }

    // TODO(mdan): This should not be necessary to update. Delete?
    mattrs->at("_num_original_outputs").set_i(num_loop_vars + num_new_chains);
    n->UpdateProperties();
    for (int i = 0; i < num_new_chains; i++) {
      g->AddEdge(new_inputs[i], 0, n, new_input_locations[i]);
    }

    // TODO(mdan): This is wasteful. Can we just mutate the original proto?
    FunctionDef modified_body = *body_graph;

    // Disable the global end-of-body barrier from the body function.
    // Because removing a node is too inefficient (would have to walk all the
    // inputs of all graph nodes), we instead clear its control dependencies.
    modified_body.mutable_node_def(body_barrier_loc)->clear_input();

    // Add extra loop vars to the body function.

    for (int i = 0; i < num_new_chains; i++) {
      // Input loop vars.
      // TODO(mdan): Double check that this doesn't clash with names in body.
      string c_name = g->NewName("acd__chainv");
      std::replace(c_name.begin(), c_name.end(), '/', '_');
      auto* new_arg = modified_body.mutable_signature()->add_input_arg();
      new_arg->set_name(c_name);
      new_arg->set_type(DT_FLOAT);

      // Output ops. These are copies of the inputs conditioned on the actual
      // control outputs.
      string c_out_name = g->NewName("acd__outchain");
      auto* new_out = modified_body.add_node_def();
      new_out->set_name(c_out_name);
      new_out->set_op("Identity");
      new_out->add_input(c_name);
      new_out->add_input(
          strings::StrCat("^", body_graph->signature().control_output(i)));
      AttrValue attr;
      attr.set_type(DT_FLOAT);
      new_out->mutable_attr()->insert({"T", attr});

      // Output loop var declarations.
      string c_ret_name = c_out_name;
      std::replace(c_ret_name.begin(), c_ret_name.end(), '/', '_');
      auto* new_out_arg = modified_body.mutable_signature()->add_output_arg();
      new_out_arg->set_name(c_ret_name);
      new_out_arg->set_type(DT_FLOAT);

      // Actual output loop vars.
      modified_body.mutable_ret()->insert(
          {c_ret_name, strings::StrCat(c_out_name, ":output:0")});
      AttrValue attr_val;
      attr_val.mutable_list()->mutable_shape();
      FunctionDef_ArgAttrs arg_attrs;
      arg_attrs.mutable_attr()->insert({"_output_shapes", attr_val});
      modified_body.mutable_arg_attr()->insert(
          {static_cast<uint32_t>(i + num_loop_vars), arg_attrs});
    }

    // Wire chain loop vars to the ops they need to condition.

    node_index.clear();
    for (int i = 0; i < modified_body.node_def_size(); i++) {
      node_index.emplace(modified_body.node_def(i).name(), i);
    }
    auto& modified_sig = modified_body.signature();
    for (int i = 0; i < num_new_chains; i++) {
      const auto& control_node =
          modified_body.node_def(node_index[modified_sig.control_output(i)]);
      for (const auto& r :
           control_node.attr().at("_res_first_used_by").list().s()) {
        NodeDef* first_node = modified_body.mutable_node_def(node_index[r]);
        // This control dependency ensures proper sequencing of stateful ops
        // upon entry into the loop body, so that they run after the ops
        // which affected the same resource in the previous iteration.
        first_node->add_input(strings::StrCat(
            "^", modified_sig.input_arg(i + num_loop_vars).name()));
      }
    }

    // Clear body function's control returns.
    modified_body.mutable_control_ret()->clear();

    // Add extra loop vars to the cond function.

    // TODO(mdan): This is wasteful. Can't we just mutate the original proto?
    string cond_name = attrs.at("cond").func().name();
    auto* cond_graph = flib_def->Find(cond_name);
    DCHECK(cond_graph != nullptr);
    FunctionDef modified_cond = *cond_graph;

    int cond_barrier_loc = -1;
    for (int i = 0, s = cond_graph->node_def_size(); i < s; i++) {
      if (cond_barrier_loc < 0) {
        const auto& node_attr = cond_graph->node_def(i).attr();
        if (node_attr.find("_acd_function_control_output") != node_attr.end()) {
          cond_barrier_loc = i;
        }
      }
    }
    if (cond_barrier_loc > 0) {
      // Disable the global end-of-body barrier from the cond function.
      // Because removing a node is too inefficient (would have to walk all the
      // inputs of all graph nodes), we instead clear its control dependencies.
      modified_cond.mutable_node_def(cond_barrier_loc)->clear_input();
    }

    for (int i = 0; i < num_new_chains; i++) {
      // Input loop vars.
      // TODO(mdan): These should gate the stateful ops in the cond.
      // Until ACD supplies the necessary information, these are dummies in this
      // function.
      string c_name = g->NewName("acd__chain");
      auto* new_arg = modified_cond.mutable_signature()->add_input_arg();
      new_arg->set_name(c_name);
      new_arg->set_type(DT_FLOAT);

      // TODO(mdan): Return values on the cond function? Most likely a bug.
      AttrValue attr_val;
      attr_val.mutable_list()->mutable_shape();
      FunctionDef_ArgAttrs arg_attrs;
      arg_attrs.mutable_attr()->insert({"_output_shapes", attr_val});
      modified_cond.mutable_arg_attr()->insert(
          {static_cast<uint32_t>(i + num_loop_vars), arg_attrs});
    }

    // Wire the new cond/body functions to the While node.

    string new_cond_name = g->NewName("acd__while_cond");
    modified_cond.mutable_signature()->set_name(new_cond_name);
    mattrs->at("cond").mutable_func()->set_name(new_cond_name);

    string new_body_name = g->NewName("acd__while_body");
    modified_body.mutable_signature()->set_name(new_body_name);
    mattrs->at("body").mutable_func()->set_name(new_body_name);

    // Commit the new functions.

    TF_RETURN_WITH_CONTEXT_IF_ERROR(
        flib_def->AddFunctionDef(modified_body,
                                 flib_def->GetStackTraces(body_name)),
        "while attaching ", new_body_name, " to flib_def");
    TF_RETURN_WITH_CONTEXT_IF_ERROR(
        flib_def->AddFunctionDef(modified_cond,
                                 flib_def->GetStackTraces(cond_name)),
        "while attaching ", new_cond_name, " to flib_def");

    // TODO(b/183666205): This should not be necessary.
    // It's unclear why adding the functions here is also required.
    // Moreover, it's unclear when graph_lib's parent is flib_def itself.
    auto* graph_lib = g->mutable_flib_def();
    if (graph_lib->default_registry() != flib_def) {
      TF_RETURN_WITH_CONTEXT_IF_ERROR(
          graph_lib->AddFunctionDef(modified_body,
                                    graph_lib->GetStackTraces(body_name)),
          "while attaching ", new_body_name, " to graph");
      TF_RETURN_WITH_CONTEXT_IF_ERROR(
          graph_lib->AddFunctionDef(modified_cond,
                                    graph_lib->GetStackTraces(cond_name)),
          "while attaching ", new_cond_name, " to graph");
    }
  }

  if (VLOG_IS_ON(1)) {
    DumpGraphToFile("control_flow_deps_to_chains_after", *g, flib_def);
  }

  return Status::OK();
}

// Note: This needs to run before functional control flow lowering, which is 10.
REGISTER_OPTIMIZATION(OptimizationPassRegistry::PRE_PLACEMENT, 9,
                      ControlFlowDepsToChainsPass);

}  // namespace tensorflow
<commit_msg>Actually add an empty shape to the attribute (it's a repeated field, not an optional one).<commit_after>/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/common_runtime/control_flow_deps_to_chains.h"
#include <cstdint>

#include "tensorflow/core/framework/attr_value.pb.h"
#include "tensorflow/core/framework/node_def.pb.h"
#include "tensorflow/core/framework/node_def_util.h"
#include "tensorflow/core/framework/op_def_builder.h"
#include "tensorflow/core/framework/tensor.pb.h"
#include "tensorflow/core/platform/errors.h"
#include "tensorflow/core/platform/strcat.h"
#include "tensorflow/core/util/dump_graph.h"

namespace tensorflow {

// TODO(mdan): Move this into Grappler - cleaner interface.
Status ControlFlowDepsToChainsPass::Run(
    const GraphOptimizationPassOptions& options) {
  VLOG(1) << "ControlFlowDepsToChainsPass::Run";

  if (options.graph == nullptr) {
    VLOG(1) << "ControlFlowDepsToChainsPass::Run Aborted";
    return Status::OK();
  }

  Graph* g = options.graph->get();
  DCHECK(g != nullptr);
  FunctionLibraryDefinition* flib_def = options.flib_def;
  DCHECK(flib_def != nullptr);

  if (VLOG_IS_ON(1)) {
    DumpGraphToFile("control_flow_deps_to_chains_before", *g, flib_def);
  }

  for (Node* n : g->nodes()) {
    if (n == nullptr) continue;
    if (!n->IsWhileNode()) continue;

    // TODO(mdan): This breaks encapsulation of Node/Graph. Is there any needed?
    // TODO(mdan): Consolidate this with AddWhileInputHack.
    NodeDef* while_node = n->mutable_def();
    const auto& attrs = while_node->attr();
    auto* mattrs = while_node->mutable_attr();

    string body_name = attrs.at("body").func().name();
    auto* body_graph = flib_def->Find(body_name);
    DCHECK(body_graph != nullptr);

    // Look for required annotations.

    if (attrs.find("_stateful_parallelism") == attrs.end()) continue;
    if (!attrs.at("_stateful_parallelism").b()) continue;
    // TODO(mdan): We don't really need this attribute.
    if (attrs.find("_num_original_outputs") == attrs.end()) continue;
    int body_barrier_loc = -1;
    std::map<string, int> node_index;
    for (int i = 0, s = body_graph->node_def_size(); i < s; i++) {
      node_index.emplace(body_graph->node_def(i).name(), i);
      if (body_barrier_loc < 0) {
        const auto& node_attr = body_graph->node_def(i).attr();
        if (node_attr.find("_acd_function_control_output") != node_attr.end()) {
          body_barrier_loc = i;
        }
      }
    }
    if (body_barrier_loc < 0) continue;
    bool ok_for_lowering = true;
    for (int i = 0; i < body_graph->control_ret_size(); i++) {
      const auto& control_node = body_graph->node_def(
          node_index[body_graph->signature().control_output(i)]);
      const auto& control_attr = control_node.attr();
      if (control_attr.find("_res_first_used_by") == control_attr.end()) {
        ok_for_lowering = false;
        break;
      }
    }
    if (!ok_for_lowering) continue;

    int num_loop_vars = body_graph->signature().input_arg_size();
    int num_new_chains = body_graph->control_ret_size();
    int num_node_inputs = while_node->input_size();

    if (!num_new_chains) continue;  // Nothing to do for stateless loops.

    // Add extra loop vars to the while node.

    // TODO(mdan): If the loop vars contains the resource, we should reuse it.
    // Note that stateful ops of resource inputs cause their resources to be
    // captured into the loop vars (through the body/cond captures). We could
    // effectively use those as chains.

    // TODO(mdan): Is there a more efficient way to do this?
    // Insert the new While node inputs: at the end of the loop vars, but before
    // any non-loop var inputs (like control dependencies). Once the initial
    // chain values are created below, they will be added to these inputs.
    for (int i = 0; i < num_new_chains; i++) {
      while_node->add_input();
    }
    for (int i = num_node_inputs - 1; i >= num_loop_vars; i--) {
      while_node->set_input(i + num_new_chains, while_node->input(i));
    }

    std::vector<Node*> new_inputs;
    std::vector<int> new_input_locations;
    // Set their name to a gensym, type to float and shape to scalar.
    for (int i = 0; i < num_new_chains; i++) {
      string c_name = g->NewName("acd__chain");

      // The initial value for the i'th chain loop var.
      NodeDef new_in;
      new_in.set_name(c_name);
      new_in.set_op("Const");
      AttrValue att_dtype;
      att_dtype.set_type(DT_FLOAT);
      new_in.mutable_attr()->insert({"dtype", att_dtype});
      AttrValue att_value;
      att_value.mutable_tensor()->set_dtype(DT_FLOAT);
      att_value.mutable_tensor()->mutable_tensor_shape();
      att_value.mutable_tensor()->add_int_val(0);
      new_in.mutable_attr()->insert({"value", att_value});
      Status status;
      new_inputs.push_back(g->AddNode(new_in, &status));
      TF_RETURN_WITH_CONTEXT_IF_ERROR(status, "while creating chain", c_name);

      int loc = num_loop_vars + i;
      new_input_locations.push_back(loc);
      while_node->set_input(loc, c_name);
      mattrs->at("T").mutable_list()->add_type(DT_FLOAT);
      mattrs->at("output_shapes").mutable_list()->add_shape();
    }

    // TODO(mdan): This should not be necessary to update. Delete?
    mattrs->at("_num_original_outputs").set_i(num_loop_vars + num_new_chains);
    n->UpdateProperties();
    for (int i = 0; i < num_new_chains; i++) {
      g->AddEdge(new_inputs[i], 0, n, new_input_locations[i]);
    }

    // TODO(mdan): This is wasteful. Can we just mutate the original proto?
    FunctionDef modified_body = *body_graph;

    // Disable the global end-of-body barrier from the body function.
    // Because removing a node is too inefficient (would have to walk all the
    // inputs of all graph nodes), we instead clear its control dependencies.
    modified_body.mutable_node_def(body_barrier_loc)->clear_input();

    // Add extra loop vars to the body function.

    for (int i = 0; i < num_new_chains; i++) {
      // Input loop vars.
      // TODO(mdan): Double check that this doesn't clash with names in body.
      string c_name = g->NewName("acd__chainv");
      std::replace(c_name.begin(), c_name.end(), '/', '_');
      auto* new_arg = modified_body.mutable_signature()->add_input_arg();
      new_arg->set_name(c_name);
      new_arg->set_type(DT_FLOAT);

      // Output ops. These are copies of the inputs conditioned on the actual
      // control outputs.
      string c_out_name = g->NewName("acd__outchain");
      auto* new_out = modified_body.add_node_def();
      new_out->set_name(c_out_name);
      new_out->set_op("Identity");
      new_out->add_input(c_name);
      new_out->add_input(
          strings::StrCat("^", body_graph->signature().control_output(i)));
      AttrValue attr;
      attr.set_type(DT_FLOAT);
      new_out->mutable_attr()->insert({"T", attr});

      // Output loop var declarations.
      string c_ret_name = c_out_name;
      std::replace(c_ret_name.begin(), c_ret_name.end(), '/', '_');
      auto* new_out_arg = modified_body.mutable_signature()->add_output_arg();
      new_out_arg->set_name(c_ret_name);
      new_out_arg->set_type(DT_FLOAT);

      // Actual output loop vars.
      modified_body.mutable_ret()->insert(
          {c_ret_name, strings::StrCat(c_out_name, ":output:0")});
      AttrValue attr_val;
      attr_val.mutable_list()->add_shape();
      FunctionDef_ArgAttrs arg_attrs;
      arg_attrs.mutable_attr()->insert({"_output_shapes", attr_val});
      modified_body.mutable_arg_attr()->insert(
          {static_cast<uint32_t>(i + num_loop_vars), arg_attrs});
    }

    // Wire chain loop vars to the ops they need to condition.

    node_index.clear();
    for (int i = 0; i < modified_body.node_def_size(); i++) {
      node_index.emplace(modified_body.node_def(i).name(), i);
    }
    auto& modified_sig = modified_body.signature();
    for (int i = 0; i < num_new_chains; i++) {
      const auto& control_node =
          modified_body.node_def(node_index[modified_sig.control_output(i)]);
      for (const auto& r :
           control_node.attr().at("_res_first_used_by").list().s()) {
        NodeDef* first_node = modified_body.mutable_node_def(node_index[r]);
        // This control dependency ensures proper sequencing of stateful ops
        // upon entry into the loop body, so that they run after the ops
        // which affected the same resource in the previous iteration.
        first_node->add_input(strings::StrCat(
            "^", modified_sig.input_arg(i + num_loop_vars).name()));
      }
    }

    // Clear body function's control returns.
    modified_body.mutable_control_ret()->clear();

    // Add extra loop vars to the cond function.

    // TODO(mdan): This is wasteful. Can't we just mutate the original proto?
    string cond_name = attrs.at("cond").func().name();
    auto* cond_graph = flib_def->Find(cond_name);
    DCHECK(cond_graph != nullptr);
    FunctionDef modified_cond = *cond_graph;

    int cond_barrier_loc = -1;
    for (int i = 0, s = cond_graph->node_def_size(); i < s; i++) {
      if (cond_barrier_loc < 0) {
        const auto& node_attr = cond_graph->node_def(i).attr();
        if (node_attr.find("_acd_function_control_output") != node_attr.end()) {
          cond_barrier_loc = i;
        }
      }
    }
    if (cond_barrier_loc > 0) {
      // Disable the global end-of-body barrier from the cond function.
      // Because removing a node is too inefficient (would have to walk all the
      // inputs of all graph nodes), we instead clear its control dependencies.
      modified_cond.mutable_node_def(cond_barrier_loc)->clear_input();
    }

    for (int i = 0; i < num_new_chains; i++) {
      // Input loop vars.
      // TODO(mdan): These should gate the stateful ops in the cond.
      // Until ACD supplies the necessary information, these are dummies in this
      // function.
      string c_name = g->NewName("acd__chain");
      auto* new_arg = modified_cond.mutable_signature()->add_input_arg();
      new_arg->set_name(c_name);
      new_arg->set_type(DT_FLOAT);

      // TODO(mdan): Return values on the cond function? Most likely a bug.
      AttrValue attr_val;
      attr_val.mutable_list()->add_shape();
      FunctionDef_ArgAttrs arg_attrs;
      arg_attrs.mutable_attr()->insert({"_output_shapes", attr_val});
      modified_cond.mutable_arg_attr()->insert(
          {static_cast<uint32_t>(i + num_loop_vars), arg_attrs});
    }

    // Wire the new cond/body functions to the While node.

    string new_cond_name = g->NewName("acd__while_cond");
    modified_cond.mutable_signature()->set_name(new_cond_name);
    mattrs->at("cond").mutable_func()->set_name(new_cond_name);

    string new_body_name = g->NewName("acd__while_body");
    modified_body.mutable_signature()->set_name(new_body_name);
    mattrs->at("body").mutable_func()->set_name(new_body_name);

    // Commit the new functions.

    TF_RETURN_WITH_CONTEXT_IF_ERROR(
        flib_def->AddFunctionDef(modified_body,
                                 flib_def->GetStackTraces(body_name)),
        "while attaching ", new_body_name, " to flib_def");
    TF_RETURN_WITH_CONTEXT_IF_ERROR(
        flib_def->AddFunctionDef(modified_cond,
                                 flib_def->GetStackTraces(cond_name)),
        "while attaching ", new_cond_name, " to flib_def");

    // TODO(b/183666205): This should not be necessary.
    // It's unclear why adding the functions here is also required.
    // Moreover, it's unclear when graph_lib's parent is flib_def itself.
    auto* graph_lib = g->mutable_flib_def();
    if (graph_lib->default_registry() != flib_def) {
      TF_RETURN_WITH_CONTEXT_IF_ERROR(
          graph_lib->AddFunctionDef(modified_body,
                                    graph_lib->GetStackTraces(body_name)),
          "while attaching ", new_body_name, " to graph");
      TF_RETURN_WITH_CONTEXT_IF_ERROR(
          graph_lib->AddFunctionDef(modified_cond,
                                    graph_lib->GetStackTraces(cond_name)),
          "while attaching ", new_cond_name, " to graph");
    }
  }

  if (VLOG_IS_ON(1)) {
    DumpGraphToFile("control_flow_deps_to_chains_after", *g, flib_def);
  }

  return Status::OK();
}

// Note: This needs to run before functional control flow lowering, which is 10.
REGISTER_OPTIMIZATION(OptimizationPassRegistry::PRE_PLACEMENT, 9,
                      ControlFlowDepsToChainsPass);

}  // namespace tensorflow
<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  $RCSfile: cacheupdatelistener.hxx,v $
 *
 *  $Revision: 1.3 $
 *
 *  last change: $Author: svesik $ $Date: 2004-04-21 11:58:02 $
 *
 *  The Contents of this file are made available subject to the terms of
 *  either of the following licenses
 *
 *         - GNU Lesser General Public License Version 2.1
 *         - Sun Industry Standards Source License Version 1.1
 *
 *  Sun Microsystems Inc., October, 2000
 *
 *  GNU Lesser General Public License Version 2.1
 *  =============================================
 *  Copyright 2000 by Sun Microsystems, Inc.
 *  901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License version 2.1, as published by the Free Software Foundation.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *  MA  02111-1307  USA
 *
 *
 *  Sun Industry Standards Source License Version 1.1
 *  =================================================
 *  The contents of this file are subject to the Sun Industry Standards
 *  Source License Version 1.1 (the "License"); You may not use this file
 *  except in compliance with the License. You may obtain a copy of the
 *  License at http://www.openoffice.org/license.html.
 *
 *  Software provided under this License is provided on an "AS IS" basis,
 *  WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
 *  WITHOUT LIMITATION, WARRANTIES THAT THE SOFTWARE IS FREE OF DEFECTS,
 *  MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE, OR NON-INFRINGING.
 *  See the License for the specific provisions governing your rights and
 *  obligations concerning the Software.
 *
 *  The Initial Developer of the Original Code is: Sun Microsystems, Inc.
 *
 *  Copyright: 2000 by Sun Microsystems, Inc.
 *
 *  All Rights Reserved.
 *
 *  Contributor(s): _______________________________________
 *
 *
 ************************************************************************/
#ifndef __FILTER_CONFIG_CACHEUPDATELISTENER_HXX_
#define __FILTER_CONFIG_CACHEUPDATELISTENER_HXX_

//_______________________________________________
// includes

#include "filtercache.hxx"

#ifndef _COM_SUN_STAR_LANG_XMULTISERVICEFACTORY_HPP_
#include <com/sun/star/lang/XMultiServiceFactory.hpp>
#endif

#ifndef _COM_SUN_STAR_LANG_XEVENTLISTENER_HPP_
#include <com/sun/star/lang/XEventListener.hpp>
#endif

#ifndef _COM_SUN_STAR_DOCUMENT_XEVENTLISTENER_HPP_
#include <com/sun/star/document/XEventListener.hpp>
#endif

#ifndef _COM_SUN_STAR_DOCUMENT_XEVENTBROADCASTER_HPP_
#include <com/sun/star/document/XEventBroadcaster.hpp>
#endif

#ifndef _SALHELPER_SINGLETONREF_HXX_
#include <salhelper/singletonref.hxx>
#endif

#ifndef _CPPUHELPER_IMPLBASE1_HXX_
#include <cppuhelper/implbase1.hxx>
#endif

//_______________________________________________
// namespace

namespace filter{
    namespace config{

//_______________________________________________
// definitions

//_______________________________________________

/** @short      implements a listener, which will update the
                global filter cache of an office, after zje office
                startup was finished.

    @descr      To perform startup of an office, the filter cache starts
                with a minimum set of properties only. After the first document
                was loaded successfully a thread will be started to fill the
                cache with all other proeprties, so it can work with the whole
                filter configuration.
 */
class CacheUpdateListener : public BaseLock // must be the first one to guarantee right initialized mutex member!
                          , public ::cppu::WeakImplHelper1< css::document::XEventListener >
{
    //-------------------------------------------
    // member

    private:

        /** @short  reference to an uno service manager, which can be used
                    to create own needed services. */
        css::uno::Reference< css::lang::XMultiServiceFactory > m_xSMGR;

        /** @short  reference(!) to the singleton filter cache implementation,
                    which should be updated by this thread. */
        ::salhelper::SingletonRef< FilterCache > m_rCache;

        /** @short  reference to the global event broadcaster, which is usde to find
                    out, when the first office document was opened successfully. */
        css::uno::Reference< css::document::XEventBroadcaster > m_xBroadcaster;

    //-------------------------------------------
    // native interface

    public:

        //---------------------------------------
        // ctor/dtor

        /** @short  initialize new instance of this class.

            @descr  It set a reference to the global filter cache singleton,
                    which should be updated here. Further it starts listening
                    on the global event broadcaster to get the information, when
                    loading of the first document was finished.

            @param  xSMGR
                    reference to a service manager, which can be used to create
                    own needed uno services.
         */
        CacheUpdateListener(const css::uno::Reference< css::lang::XMultiServiceFactory >& xSMGR);

        //---------------------------------------

        /** @short  standard dtor.
         */
        virtual ~CacheUpdateListener();

    //-------------------------------------------
    // uno interface

    public:

        //---------------------------------------
        // document.XEventListener

        virtual void SAL_CALL notifyEvent(const css::document::EventObject& aEvent)
            throw(css::uno::RuntimeException);

        //---------------------------------------
        // lang.XEventListener
        virtual void SAL_CALL disposing(const css::lang::EventObject& aEvent)
            throw(css::uno::RuntimeException);
};

    } // namespace config
} // namespace filter

#endif // __FILTER_CONFIG_CACHEUPDATELISTENER_HXX_
<commit_msg>INTEGRATION: CWS os12 (1.2.26); FILE MERGED 2004/03/08 10:53:59 as 1.2.26.2: #114059# use clone/takeover instead of configuration listener 2004/03/05 13:53:58 as 1.2.26.1: #114059#, #115733# start implementation of listener mode, read old typedetection file, fix build dependencies<commit_after>/*************************************************************************
 *
 *  $RCSfile: cacheupdatelistener.hxx,v $
 *
 *  $Revision: 1.4 $
 *
 *  last change: $Author: obo $ $Date: 2004-04-29 13:39:26 $
 *
 *  The Contents of this file are made available subject to the terms of
 *  either of the following licenses
 *
 *         - GNU Lesser General Public License Version 2.1
 *         - Sun Industry Standards Source License Version 1.1
 *
 *  Sun Microsystems Inc., October, 2000
 *
 *  GNU Lesser General Public License Version 2.1
 *  =============================================
 *  Copyright 2000 by Sun Microsystems, Inc.
 *  901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License version 2.1, as published by the Free Software Foundation.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *  MA  02111-1307  USA
 *
 *
 *  Sun Industry Standards Source License Version 1.1
 *  =================================================
 *  The contents of this file are subject to the Sun Industry Standards
 *  Source License Version 1.1 (the "License"); You may not use this file
 *  except in compliance with the License. You may obtain a copy of the
 *  License at http://www.openoffice.org/license.html.
 *
 *  Software provided under this License is provided on an "AS IS" basis,
 *  WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
 *  WITHOUT LIMITATION, WARRANTIES THAT THE SOFTWARE IS FREE OF DEFECTS,
 *  MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE, OR NON-INFRINGING.
 *  See the License for the specific provisions governing your rights and
 *  obligations concerning the Software.
 *
 *  The Initial Developer of the Original Code is: Sun Microsystems, Inc.
 *
 *  Copyright: 2000 by Sun Microsystems, Inc.
 *
 *  All Rights Reserved.
 *
 *  Contributor(s): _______________________________________
 *
 *
 ************************************************************************/
#ifndef __FILTER_CONFIG_CACHEUPDATELISTENER_HXX_
#define __FILTER_CONFIG_CACHEUPDATELISTENER_HXX_

//_______________________________________________
// includes

#include "filtercache.hxx"

#ifndef _COM_SUN_STAR_LANG_XMULTISERVICEFACTORY_HPP_
#include <com/sun/star/lang/XMultiServiceFactory.hpp>
#endif

#ifndef _COM_SUN_STAR_LANG_XEVENTLISTENER_HPP_
#include <com/sun/star/lang/XEventListener.hpp>
#endif

#ifndef _COM_SUN_STAR_UTIL_XCHANGESLISTENER_HPP_
#include <com/sun/star/util/XChangesListener.hpp>
#endif

#ifndef _SALHELPER_SINGLETONREF_HXX_
#include <salhelper/singletonref.hxx>
#endif

#ifndef _CPPUHELPER_IMPLBASE1_HXX_
#include <cppuhelper/implbase1.hxx>
#endif

//_______________________________________________
// namespace

namespace filter{
    namespace config{

//_______________________________________________
// definitions

//_______________________________________________

/** @short      implements a listener, which will update the
                global filter cache, if the underlying configuration
                wa changed by other processes.
 */
class CacheUpdateListener : public BaseLock // must be the first one to guarantee right initialized mutex member!
                          , public ::cppu::WeakImplHelper1< css::util::XChangesListener >
{
    //-------------------------------------------
    // member

    private:

        /** @short  reference to an uno service manager, which can be used
                    to create own needed services. */
        css::uno::Reference< css::lang::XMultiServiceFactory > m_xSMGR;

        /** @short  reference to the singleton(!) filter cache implementation,
                    which should be updated by this thread. */
        ::salhelper::SingletonRef< FilterCache > m_rCache;

    //-------------------------------------------
    // native interface

    public:

        //---------------------------------------
        // ctor/dtor

        /** @short  initialize new instance of this class.

            @descr  Listening wont be started here. It can be done
                    by calling startListening() on this instance.

            @see    startListening()

            @param  xSMGR
                    reference to a service manager, which can be used to create
                    own needed uno services.
         */
        CacheUpdateListener(const css::uno::Reference< css::lang::XMultiServiceFactory >& xSMGR);

        //---------------------------------------

        /** @short  standard dtor.
         */
        virtual ~CacheUpdateListener();

        //---------------------------------------

        /** @short  starts listening on the given configuration access.

            @descr  There wont be a check, if this instance is already a listener
                    on the given configuration access! Such things must be checked
                    by the outside code.

                    => startListening() and stopListening() must be called in pairs.
                    But calling of stopListening() isnt required in real.
                    If the configuration access dies, the listener dies automaticly too.

            @param  xConfigAccess
                    the configuration access, where this instance should listen for changes.
         */
        virtual void startListening(const css::uno::Reference< css::uno::XInterface >& xConfigAccess);

        //---------------------------------------

        /** @short  stop listening on the given configuration access.

            @descr  There wont be a check, if this instance is a listener
                    on the given configuration access or not! Such things must be checked
                    by the outside code.

                    => startListening() and stopListening() must be called in pairs.
                    But calling of stopListening() isnt required in real.
                    If the configuration access dies, the listener dies automaticly too.

            @param  xConfigAccess
                    the configuration access, where this instance should stop listening.
         */
        virtual void stopListening(const css::uno::Reference< css::uno::XInterface >& xConfigAccess);

    //-------------------------------------------
    // uno interface

    public:

        //---------------------------------------
        // XChangesListener

        virtual void SAL_CALL changesOccurred(const css::util::ChangesEvent& aEvent)
            throw(css::uno::RuntimeException);

        //---------------------------------------
        // lang.XEventListener
        virtual void SAL_CALL disposing(const css::lang::EventObject& aEvent)
            throw(css::uno::RuntimeException);
};

    } // namespace config
} // namespace filter

#endif // __FILTER_CONFIG_CACHEUPDATELISTENER_HXX_
<|endoftext|>


<commit_before>#include "regiondataremote.h"

#include "regionmatrixproxy.h"
#include "workerthread.h"

using namespace petabricks;
using namespace petabricks::RegionDataRemoteMessage;

RegionDataRemote::RegionDataRemote(const int dimensions, const IndexT* size, const RegionDataRemoteObjectPtr remoteObject) {
  init(dimensions, size, remoteObject);
}

RegionDataRemote::RegionDataRemote(const int dimensions, const IndexT* size, RemoteHostPtr host) {
  init(dimensions, size, new RegionDataRemoteObject());

  // InitialMsg
  size_t size_sz = _D * sizeof(IndexT);
  size_t msg_len = sizeof(CreateRegionDataInitialMessage) + size_sz;

  char buf[msg_len];
  CreateRegionDataInitialMessage* msg = (CreateRegionDataInitialMessage*)buf;
  msg->type = MessageTypes::CREATEREGIONDATA;
  msg->dimensions = _D;
  memcpy(msg->size, size, size_sz);

  host->createRemoteObject(_remoteObject.asPtr(), &RegionMatrixProxy::genRemote, buf, msg_len);
}

RegionDataRemote::RegionDataRemote(const int dimensions, const IndexT* size, RemoteHost& host, const MessageType initialMessageType, const EncodedPtr encodePtr) {
  init(dimensions, size, new RegionDataRemoteObject());

  // InitialMsg
  EncodedPtrInitialMessage msg;
  msg.type = initialMessageType;
  msg.encodedPtr = encodePtr;
  int len = sizeof(EncodedPtrInitialMessage);
  host.createRemoteObject(_remoteObject.asPtr(), &RegionMatrixProxy::genRemote, &msg, len);
}

void RegionDataRemote::init(const int dimensions, const IndexT* size, const RegionDataRemoteObjectPtr remoteObject) {
  _D = dimensions;
  _type = RegionDataTypes::REGIONDATAREMOTE;
  _remoteObject = remoteObject;

  memcpy(_size, size, sizeof(IndexT) * _D);
}

int RegionDataRemote::allocData() {
  void* data;
  size_t len;
  this->fetchData(0, MessageTypes::ALLOCDATA, 0, &data, &len);
  AllocDataReplyMessage* reply = (AllocDataReplyMessage*)data;

  ElementT result = reply->result;
  free(reply);
  return result;
}

void RegionDataRemote::randomize() {
  void* data;
  size_t len;
  this->fetchData(0, MessageTypes::RANDOMIZEDATA, 0, &data, &len);

  free(data);
}

IRegionCachePtr RegionDataRemote::cacheGenerator() const {
  IndexT multipliers[_D];
  multipliers[0] = 1;
  for (int i = 1; i < _D; i++) {
    multipliers[i] = multipliers[i - 1] * _size[i - 1];
  }

  return new RegionDataRemoteCache(this, _D, multipliers);
}

IRegionCachePtr RegionDataRemote::cache() const {
#ifdef DISTRIBUTED_CACHE
  if (WorkerThread::self()) {
    return WorkerThread::self()->cache()->get(this);
  } else {
    // in listening loop
    return NULL;
  }
#else
  return NULL;
#endif
}

void RegionDataRemote::invalidateCache() {
#ifdef DISTRIBUTED_CACHE
  if (cache()) {
    cache()->invalidate();
  }
#endif
}

ElementT RegionDataRemote::readCell(const IndexT* coord) const {
#ifdef DISTRIBUTED_CACHE
  if (cache()) {
    return cache()->readCell(coord);
  } else {
    return readNoCache(coord);
  }
#else
  return readNoCache(coord);
#endif
}

ElementT RegionDataRemote::readNoCache(const IndexT* coord) const {
  size_t coord_sz = _D * sizeof(IndexT);
  size_t msg_len = sizeof(ReadCellMessage) + coord_sz;

  char buf[msg_len];
  ReadCellMessage* msg = (ReadCellMessage*)buf;
  memcpy(msg->coord, coord, coord_sz);

  void* data;
  size_t len;
  this->fetchData(buf, MessageTypes::READCELL, msg_len, &data, &len);
  ReadCellReplyMessage* reply = (ReadCellReplyMessage*)data;
  ElementT value = reply->value;
  free(reply);
  return value;
}

void RegionDataRemote::readByCache(void* request, size_t request_len, void* reply, size_t &/*reply_len*/) const {
  void* data;
  size_t len;
  this->fetchData(request, MessageTypes::READCELLCACHE, request_len, &data, &len);
  ReadCellCacheReplyMessage* r = (ReadCellCacheReplyMessage*)data;

  RegionDataRemoteCacheLine* cacheLine = (RegionDataRemoteCacheLine*)reply;
  cacheLine->start = r->start;
  cacheLine->end = r->end;
  memcpy(cacheLine->base, r->values, sizeof(ElementT) * (r->end + 1));
  free(r);
}

void RegionDataRemote::writeCell(const IndexT* coord, ElementT value) {
#ifdef DISTRIBUTED_CACHE
  if (cache()) {
    cache()->writeCell(coord, value);
  } else {
    writeNoCache(coord, value);
  }
#else
  writeNoCache(coord, value);
#endif
}

void RegionDataRemote::writeNoCache(const IndexT* coord, ElementT value) {
  size_t coord_sz = _D * sizeof(IndexT);
  size_t msg_len = sizeof(WriteCellMessage) + coord_sz;

  char buf[msg_len];
  WriteCellMessage* msg = (WriteCellMessage*)buf;
  msg->value = value;
  memcpy(msg->coord, coord, coord_sz);

  //JTRACE("write")(_D)(sizeof(IndexT))(coord_sz)(msg_len);

  void* data;
  size_t len;
  this->fetchData(buf, MessageTypes::WRITECELL, msg_len, &data, &len);
  WriteCellReplyMessage* reply = (WriteCellReplyMessage*)data;

  free(reply);
}

void RegionDataRemote::writeByCache(const IndexT* coord, ElementT value) const {
  size_t coord_sz = _D * sizeof(IndexT);
  size_t msg_len = sizeof(WriteCellMessage) + coord_sz;

  char buf[msg_len];
  WriteCellMessage* msg = (WriteCellMessage*)buf;
  msg->value = value;
  memcpy(msg->coord, coord, coord_sz);

  void* data;
  size_t len;
  this->fetchData(msg, MessageTypes::WRITECELL, msg_len, &data, &len);
  WriteCellReplyMessage* reply = (WriteCellReplyMessage*)data;

  free(reply);
}

void RegionDataRemote::copyToScratchMatrixStorage(CopyToMatrixStorageMessage* origMetadata, size_t len, MatrixStoragePtr scratchStorage, MatrixRegionMetadata* scratchMetadata) const {
  void* data;
  size_t replyLen;
  this->fetchData(origMetadata, MessageTypes::TOSCRATCHSTORAGE, len, &data, &replyLen);

  CopyToMatrixStorageReplyMessage* reply = (CopyToMatrixStorageReplyMessage*)data;
  if (scratchMetadata == 0) {
    JASSERT(reply->count == scratchStorage->count());
    memcpy(scratchStorage->data(), reply->storage, sizeof(ElementT) * reply->count);

  } else {
    UNIMPLEMENTED();

  }
  free(reply);
}

void RegionDataRemote::copyFromScratchMatrixStorage(CopyFromMatrixStorageMessage* metadata, size_t size) const {
  void* data;
  size_t len;
  this->fetchData(metadata, MessageTypes::FROMSCRATCHSTORAGE, size, &data, &len);
  free(data);
}

DataHostPidList RegionDataRemote::hosts(IndexT* begin, IndexT* end) {
  GetHostListMessage msg;
  memcpy(msg.begin, begin, _D * sizeof(IndexT));
  memcpy(msg.end, end, _D * sizeof(IndexT));

  void* data;
  size_t len;
  this->fetchData(&msg, MessageTypes::GETHOSTLIST, sizeof(GetHostListMessage), &data, &len);
  GetHostListReplyMessage* reply = (GetHostListReplyMessage*)data;

  DataHostPidList list;
  for (int i = 0; i < reply->numHosts; i++) {
    list.push_back(reply->hosts[i]);
  }

  free(reply);
  return list;
}

RemoteHostPtr RegionDataRemote::host() {
  return _remoteObject->host();
}

UpdateHandlerChainReplyMessage RegionDataRemote::updateHandlerChain(UpdateHandlerChainMessage& msg) {
  msg.numHops += 1;

  void* data;
  size_t len;
  this->fetchData(&msg, MessageTypes::UPDATEHANDLERCHAIN, sizeof(UpdateHandlerChainMessage), &data, &len);
  UpdateHandlerChainReplyMessage* _reply = (UpdateHandlerChainReplyMessage*)data;

  UpdateHandlerChainReplyMessage reply = *_reply;
  free(_reply);
  return reply;
}

UpdateHandlerChainReplyMessage RegionDataRemote::updateHandlerChain() {
  UpdateHandlerChainMessage msg;
  msg.requester = HostPid::self();
  msg.numHops = 0;
  return this->updateHandlerChain(msg);
}

void RegionDataRemote::fetchData(const void* msg, MessageType type, size_t len, void** responseData, size_t* responseLen) const {

  *responseData = 0;
  *responseLen = 0;

  size_t dataLen = sizeof(GeneralMessageHeader) + len;

  GeneralMessageHeader* header = (GeneralMessageHeader*)malloc(dataLen);
  header->isForwardMessage = false;
  header->type = type;
  header->contentOffset = sizeof(GeneralMessageHeader);
  header->responseData = reinterpret_cast<EncodedPtr>(responseData);
  header->responseLen = reinterpret_cast<EncodedPtr>(responseLen);

  memcpy(header->content(), msg, len);

  JLOCKSCOPE(*_remoteObject);
  _remoteObject->send(header, dataLen);
  free(header);

  // wait for the data
  while (*responseData == 0 || *responseLen == 0) {
    _remoteObject->waitMsgMu();
  }
}

void RegionDataRemote::onRecv(const void* data, size_t len) {
  const BaseMessageHeader* base = (const BaseMessageHeader*)data;
  if (base->isForwardMessage) {
    const ForwardMessageHeader* header = (const ForwardMessageHeader*)data;
    IRegionReplyProxy* proxy = reinterpret_cast<IRegionReplyProxy*>(header->callback);
    proxy->processReplyMsg(base, len);

  } else {
    const GeneralMessageHeader* header = (const GeneralMessageHeader*)data;

    const void** responseData = reinterpret_cast<const void**>(header->responseData);
    size_t* responseLen = reinterpret_cast<size_t*>(header->responseLen);

    size_t sz = len - base->contentOffset;
    void* msg = malloc(sz);
    memcpy(msg, base->content(), sz);

    *responseData = msg;
    *responseLen = sz;
  }
}

// Process remote messages
void RegionDataRemote::forwardMessage(const BaseMessageHeader* base, size_t baseLen, IRegionReplyProxy* caller) {
  size_t len = sizeof(ForwardMessageHeader) + baseLen;

  ForwardMessageHeader* data = (ForwardMessageHeader*)malloc(len);
  data->isForwardMessage = true;
  data->type = base->type;
  data->contentOffset = sizeof(ForwardMessageHeader) + base->contentOffset;
  data->callback = reinterpret_cast<EncodedPtr>(caller);

  memcpy(data->next(), base, baseLen);

  _remoteObject->send(data, len);
  free(data);
}

void RegionDataRemote::processReadCellMsg(const BaseMessageHeader* base, size_t baseLen, IRegionReplyProxy* caller) {
  this->forwardMessage(base, baseLen, caller);
}

void RegionDataRemote::processWriteCellMsg(const BaseMessageHeader* base, size_t baseLen, IRegionReplyProxy* caller) {
  this->forwardMessage(base, baseLen, caller);
}

void RegionDataRemote::processAllocDataMsg(const BaseMessageHeader* base, size_t baseLen, IRegionReplyProxy* caller) {
  this->forwardMessage(base, baseLen, caller);
}

void RegionDataRemote::processRandomizeDataMsg(const BaseMessageHeader* base, size_t baseLen, IRegionReplyProxy* caller) {
  this->forwardMessage(base, baseLen, caller);
}

void RegionDataRemote::processUpdateHandlerChainMsg(const BaseMessageHeader* base, size_t baseLen, IRegionReplyProxy* caller, RegionDataIPtr) {
  UpdateHandlerChainMessage* msg = (UpdateHandlerChainMessage*)base->content();
  msg->numHops++;
  this->forwardMessage(base, baseLen, caller);
}

void RegionDataRemote::processGetHostListMsg(const BaseMessageHeader* base, size_t baseLen, IRegionReplyProxy* caller) {
  this->forwardMessage(base, baseLen, caller);
}

void RegionDataRemote::processGetMatrixStorageMsg(const BaseMessageHeader* base, size_t baseLen, IRegionReplyProxy* caller) {
  this->forwardMessage(base, baseLen, caller);
}

RemoteObjectPtr RegionDataRemote::genRemote() {
  return new RegionDataRemoteObject();
}

<commit_msg>bugfix?<commit_after>#include "regiondataremote.h"

#include "regionmatrixproxy.h"
#include "workerthread.h"

using namespace petabricks;
using namespace petabricks::RegionDataRemoteMessage;

RegionDataRemote::RegionDataRemote(const int dimensions, const IndexT* size, const RegionDataRemoteObjectPtr remoteObject) {
  init(dimensions, size, remoteObject);
}

RegionDataRemote::RegionDataRemote(const int dimensions, const IndexT* size, RemoteHostPtr host) {
  init(dimensions, size, new RegionDataRemoteObject());

  // InitialMsg
  size_t size_sz = _D * sizeof(IndexT);
  size_t msg_len = sizeof(CreateRegionDataInitialMessage) + size_sz;

  char buf[msg_len];
  CreateRegionDataInitialMessage* msg = (CreateRegionDataInitialMessage*)buf;
  msg->type = MessageTypes::CREATEREGIONDATA;
  msg->dimensions = _D;
  memcpy(msg->size, size, size_sz);

  host->createRemoteObject(_remoteObject.asPtr(), &RegionMatrixProxy::genRemote, buf, msg_len);
}

RegionDataRemote::RegionDataRemote(const int dimensions, const IndexT* size, RemoteHost& host, const MessageType initialMessageType, const EncodedPtr encodePtr) {
  init(dimensions, size, new RegionDataRemoteObject());

  // InitialMsg
  EncodedPtrInitialMessage msg;
  msg.type = initialMessageType;
  msg.encodedPtr = encodePtr;
  int len = sizeof(EncodedPtrInitialMessage);
  host.createRemoteObject(_remoteObject.asPtr(), &RegionMatrixProxy::genRemote, &msg, len);
}

void RegionDataRemote::init(const int dimensions, const IndexT* size, const RegionDataRemoteObjectPtr remoteObject) {
  _D = dimensions;
  _type = RegionDataTypes::REGIONDATAREMOTE;
  _remoteObject = remoteObject;

  memcpy(_size, size, sizeof(IndexT) * _D);
}

int RegionDataRemote::allocData() {
  void* data;
  size_t len;
  this->fetchData(0, MessageTypes::ALLOCDATA, 0, &data, &len);
  AllocDataReplyMessage* reply = (AllocDataReplyMessage*)data;

  ElementT result = reply->result;
  free(reply);
  return result;
}

void RegionDataRemote::randomize() {
  void* data;
  size_t len;
  this->fetchData(0, MessageTypes::RANDOMIZEDATA, 0, &data, &len);

  free(data);
}

IRegionCachePtr RegionDataRemote::cacheGenerator() const {
  IndexT multipliers[_D];
  multipliers[0] = 1;
  for (int i = 1; i < _D; i++) {
    multipliers[i] = multipliers[i - 1] * _size[i - 1];
  }

  return new RegionDataRemoteCache(this, _D, multipliers);
}

IRegionCachePtr RegionDataRemote::cache() const {
#ifdef DISTRIBUTED_CACHE
  if (WorkerThread::self()) {
    return WorkerThread::self()->cache()->get(this);
  } else {
    // in listening loop
    return NULL;
  }
#else
  return NULL;
#endif
}

void RegionDataRemote::invalidateCache() {
#ifdef DISTRIBUTED_CACHE
  if (cache()) {
    cache()->invalidate();
  }
#endif
}

ElementT RegionDataRemote::readCell(const IndexT* coord) const {
#ifdef DISTRIBUTED_CACHE
  if (cache()) {
    return cache()->readCell(coord);
  } else {
    return readNoCache(coord);
  }
#else
  return readNoCache(coord);
#endif
}

ElementT RegionDataRemote::readNoCache(const IndexT* coord) const {
  size_t coord_sz = _D * sizeof(IndexT);
  size_t msg_len = sizeof(ReadCellMessage) + coord_sz;

  char buf[msg_len];
  ReadCellMessage* msg = (ReadCellMessage*)buf;
  memcpy(msg->coord, coord, coord_sz);

  void* data;
  size_t len;
  this->fetchData(buf, MessageTypes::READCELL, msg_len, &data, &len);
  ReadCellReplyMessage* reply = (ReadCellReplyMessage*)data;
  ElementT value = reply->value;
  free(reply);
  return value;
}

void RegionDataRemote::readByCache(void* request, size_t request_len, void* reply, size_t &/*reply_len*/) const {
  void* data;
  size_t len;
  this->fetchData(request, MessageTypes::READCELLCACHE, request_len, &data, &len);
  ReadCellCacheReplyMessage* r = (ReadCellCacheReplyMessage*)data;

  RegionDataRemoteCacheLine* cacheLine = (RegionDataRemoteCacheLine*)reply;
  cacheLine->start = r->start;
  cacheLine->end = r->end;
  memcpy(cacheLine->base, r->values, sizeof(ElementT) * (r->end + 1));
  free(r);
}

void RegionDataRemote::writeCell(const IndexT* coord, ElementT value) {
#ifdef DISTRIBUTED_CACHE
  if (cache()) {
    cache()->writeCell(coord, value);
  } else {
    writeNoCache(coord, value);
  }
#else
  writeNoCache(coord, value);
#endif
}

void RegionDataRemote::writeNoCache(const IndexT* coord, ElementT value) {
  size_t coord_sz = _D * sizeof(IndexT);
  size_t msg_len = sizeof(WriteCellMessage) + coord_sz;

  char buf[msg_len];
  WriteCellMessage* msg = (WriteCellMessage*)buf;
  msg->value = value;
  memcpy(msg->coord, coord, coord_sz);

  //JTRACE("write")(_D)(sizeof(IndexT))(coord_sz)(msg_len);

  void* data;
  size_t len;
  this->fetchData(buf, MessageTypes::WRITECELL, msg_len, &data, &len);
  WriteCellReplyMessage* reply = (WriteCellReplyMessage*)data;

  free(reply);
}

void RegionDataRemote::writeByCache(const IndexT* coord, ElementT value) const {
  size_t coord_sz = _D * sizeof(IndexT);
  size_t msg_len = sizeof(WriteCellMessage) + coord_sz;

  char buf[msg_len];
  WriteCellMessage* msg = (WriteCellMessage*)buf;
  msg->value = value;
  memcpy(msg->coord, coord, coord_sz);

  void* data;
  size_t len;
  this->fetchData(msg, MessageTypes::WRITECELL, msg_len, &data, &len);
  WriteCellReplyMessage* reply = (WriteCellReplyMessage*)data;

  free(reply);
}

void RegionDataRemote::copyToScratchMatrixStorage(CopyToMatrixStorageMessage* origMetadata, size_t len, MatrixStoragePtr scratchStorage, MatrixRegionMetadata* scratchMetadata) const {
  void* data;
  size_t replyLen;
  this->fetchData(origMetadata, MessageTypes::TOSCRATCHSTORAGE, len, &data, &replyLen);

  CopyToMatrixStorageReplyMessage* reply = (CopyToMatrixStorageReplyMessage*)data;
  if (scratchMetadata == 0) {
    JASSERT(reply->count == scratchStorage->count());
    memcpy(scratchStorage->data(), reply->storage, sizeof(ElementT) * reply->count);

  } else {
    UNIMPLEMENTED();

  }
  free(reply);
}

void RegionDataRemote::copyFromScratchMatrixStorage(CopyFromMatrixStorageMessage* metadata, size_t size) const {
  void* data;
  size_t len;
  this->fetchData(metadata, MessageTypes::FROMSCRATCHSTORAGE, size, &data, &len);
  free(data);
}

DataHostPidList RegionDataRemote::hosts(IndexT* begin, IndexT* end) {
  GetHostListMessage msg;
  memcpy(msg.begin, begin, _D * sizeof(IndexT));
  memcpy(msg.end, end, _D * sizeof(IndexT));

  void* data;
  size_t len;
  this->fetchData(&msg, MessageTypes::GETHOSTLIST, sizeof(GetHostListMessage), &data, &len);
  GetHostListReplyMessage* reply = (GetHostListReplyMessage*)data;

  DataHostPidList list;
  for (int i = 0; i < reply->numHosts; i++) {
    list.push_back(reply->hosts[i]);
  }

  free(reply);
  return list;
}

RemoteHostPtr RegionDataRemote::host() {
  return _remoteObject->host();
}

UpdateHandlerChainReplyMessage RegionDataRemote::updateHandlerChain(UpdateHandlerChainMessage& msg) {
  msg.numHops += 1;

  void* data;
  size_t len;
  this->fetchData(&msg, MessageTypes::UPDATEHANDLERCHAIN, sizeof(UpdateHandlerChainMessage), &data, &len);
  UpdateHandlerChainReplyMessage* _reply = (UpdateHandlerChainReplyMessage*)data;

  UpdateHandlerChainReplyMessage reply = *_reply;
  free(_reply);
  return reply;
}

UpdateHandlerChainReplyMessage RegionDataRemote::updateHandlerChain() {
  UpdateHandlerChainMessage msg;
  msg.requester = HostPid::self();
  msg.numHops = 0;
  return this->updateHandlerChain(msg);
}

void RegionDataRemote::fetchData(const void* msg, MessageType type, size_t len, void** responseData, size_t* responseLen) const {

  *responseData = 0;
  *responseLen = 0;

  size_t dataLen = sizeof(GeneralMessageHeader) + len;

  GeneralMessageHeader* header = (GeneralMessageHeader*)malloc(dataLen);
  header->isForwardMessage = false;
  header->type = type;
  header->contentOffset = sizeof(GeneralMessageHeader);
  header->responseData = reinterpret_cast<EncodedPtr>(responseData);
  header->responseLen = reinterpret_cast<EncodedPtr>(responseLen);

  memcpy(header->content(), msg, len);

  _remoteObject->send(header, dataLen);
  free(header);

  JLOCKSCOPE(*_remoteObject);
  // wait for the data
  while (*responseData == 0 || *responseLen == 0) {
    _remoteObject->waitMsgMu();
  }
}

void RegionDataRemote::onRecv(const void* data, size_t len) {
  const BaseMessageHeader* base = (const BaseMessageHeader*)data;
  if (base->isForwardMessage) {
    const ForwardMessageHeader* header = (const ForwardMessageHeader*)data;
    IRegionReplyProxy* proxy = reinterpret_cast<IRegionReplyProxy*>(header->callback);
    proxy->processReplyMsg(base, len);

  } else {
    const GeneralMessageHeader* header = (const GeneralMessageHeader*)data;

    const void** responseData = reinterpret_cast<const void**>(header->responseData);
    size_t* responseLen = reinterpret_cast<size_t*>(header->responseLen);

    size_t sz = len - base->contentOffset;
    void* msg = malloc(sz);
    memcpy(msg, base->content(), sz);

    *responseData = msg;
    *responseLen = sz;
  }
}

// Process remote messages
void RegionDataRemote::forwardMessage(const BaseMessageHeader* base, size_t baseLen, IRegionReplyProxy* caller) {
  size_t len = sizeof(ForwardMessageHeader) + baseLen;

  ForwardMessageHeader* data = (ForwardMessageHeader*)malloc(len);
  data->isForwardMessage = true;
  data->type = base->type;
  data->contentOffset = sizeof(ForwardMessageHeader) + base->contentOffset;
  data->callback = reinterpret_cast<EncodedPtr>(caller);

  memcpy(data->next(), base, baseLen);

  _remoteObject->send(data, len);
  free(data);
}

void RegionDataRemote::processReadCellMsg(const BaseMessageHeader* base, size_t baseLen, IRegionReplyProxy* caller) {
  this->forwardMessage(base, baseLen, caller);
}

void RegionDataRemote::processWriteCellMsg(const BaseMessageHeader* base, size_t baseLen, IRegionReplyProxy* caller) {
  this->forwardMessage(base, baseLen, caller);
}

void RegionDataRemote::processAllocDataMsg(const BaseMessageHeader* base, size_t baseLen, IRegionReplyProxy* caller) {
  this->forwardMessage(base, baseLen, caller);
}

void RegionDataRemote::processRandomizeDataMsg(const BaseMessageHeader* base, size_t baseLen, IRegionReplyProxy* caller) {
  this->forwardMessage(base, baseLen, caller);
}

void RegionDataRemote::processUpdateHandlerChainMsg(const BaseMessageHeader* base, size_t baseLen, IRegionReplyProxy* caller, RegionDataIPtr) {
  UpdateHandlerChainMessage* msg = (UpdateHandlerChainMessage*)base->content();
  msg->numHops++;
  this->forwardMessage(base, baseLen, caller);
}

void RegionDataRemote::processGetHostListMsg(const BaseMessageHeader* base, size_t baseLen, IRegionReplyProxy* caller) {
  this->forwardMessage(base, baseLen, caller);
}

void RegionDataRemote::processGetMatrixStorageMsg(const BaseMessageHeader* base, size_t baseLen, IRegionReplyProxy* caller) {
  this->forwardMessage(base, baseLen, caller);
}

RemoteObjectPtr RegionDataRemote::genRemote() {
  return new RegionDataRemoteObject();
}

<|endoftext|>


<commit_before><commit_msg>add MAYACAL<commit_after><|endoftext|>