<commit_before>//=======================================================================
// Copyright Baptiste Wicht 2015-2017.
// Distributed under the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <fstream>
#include <sstream>

#include <dirent.h>

#include "dataset.hpp"
#include "config.hpp"
#include "utils.hpp"

namespace {

void read_word_labels(spot_dataset& dataset, const std::string& path) {
    std::ifstream word_labels_stream(path + "/ground_truth/word_labels.txt");

    while (!word_labels_stream.eof()) {
        std::string image_name;
        word_labels_stream >> image_name;

        std::string label;
        word_labels_stream >> label;

        if (image_name.empty() || label.empty()) {
            continue;
        }

        std::istringstream ss(label);
        std::string token;

        while (std::getline(ss, token, '-')) {
            dataset.word_labels[image_name].push_back(token);
        }
    }
}

void read_word_labels_ak(spot_dataset& dataset, const std::string& path) {
    std::ifstream word_labels_stream(path + "/ground_truth/word_labels.txt");

    while (!word_labels_stream.eof()) {
        std::string image_name;
        word_labels_stream >> image_name;

        std::string label;
        word_labels_stream >> label;

        if (image_name.empty() || label.empty()) {
            continue;
        }

        // Get rid of AK crap
        if (label == "N/A" || label == "." || label == "," || label == "\"" || label == "-") {
            continue;
        }

        std::transform(label.begin(), label.end(), label.begin(), ::tolower);

        for(char c : label){
            dataset.word_labels[image_name].push_back(std::string() + c);
        }
    }
}

void read_line_transcriptions(spot_dataset& dataset, const std::string& path) {
    std::ifstream line_transcriptions_stream(path + "/ground_truth/transcription.txt");

    while (!line_transcriptions_stream.eof()) {
        std::string image_name;
        line_transcriptions_stream >> image_name;

        std::string label;
        line_transcriptions_stream >> label;

        if (image_name.empty() || label.empty()) {
            continue;
        }

        std::istringstream ss(label);
        std::string word_token;

        while (std::getline(ss, word_token, '|')) {
            std::vector<std::string> word_labels;

            std::istringstream ss2(word_token);
            std::string token;

            while (std::getline(ss2, token, '-')) {
                word_labels.push_back(token);
            }

            dataset.line_transcriptions[image_name].emplace_back(std::move(word_labels));
        }
    }
}

void read_images(std::unordered_map<std::string, cv::Mat>& map, const std::string& file_path) {
    std::cout << "Read images from '" << file_path << "'" << std::endl;

    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 3 || file_name.find(".png") != file_name.size() - 4) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        map[file_name] = cv::imread(full_name, CV_LOAD_IMAGE_ANYDEPTH);

        if (!map[file_name].data) {
            std::cout << "Impossible to read image " << full_name << std::endl;
        }
    }
}

void read_images_ak(const config& conf, std::unordered_map<std::string, cv::Mat>& map, const std::string& file_path, const std::string& sub) {
    std::cout << "Read images from '" << file_path << "/" << sub << "'" << std::endl;

    auto full_path = file_path + "/" + sub;

    struct dirent* entry;
    auto dir = opendir(full_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 3 || file_name.find(".png") != file_name.size() - 4) {
            continue;
        }

        std::string full_name(full_path + "/" + file_name);

        auto key = sub + "/" + file_name;

        cv::Mat base_image = cv::imread(full_name, CV_LOAD_IMAGE_ANYDEPTH);

        if (conf.method == Method::Patches) {
            if (base_image.size().height == HEIGHT) {
                map[key] = base_image;
            } else {
                float ratio = float(HEIGHT) / base_image.size().height;

                cv::Mat scaled_normalized(
                    cv::Size(static_cast<size_t>(base_image.size().width * ratio), HEIGHT),
                    CV_8U);
                cv::resize(base_image, scaled_normalized, scaled_normalized.size(), cv::INTER_AREA);
                cv::adaptiveThreshold(scaled_normalized, map[key], 255, CV_ADAPTIVE_THRESH_MEAN_C, CV_THRESH_BINARY, 7, 2);
            }
        } else {
            map[key] = base_image;
        }

        if (!map[key].data) {
            std::cout << "Impossible to read image " << full_name << std::endl;
        }
    }
}

void read_line_images(spot_dataset& dataset, const std::string& path) {
    read_images(dataset.line_images, path + "/data/line_images_normalized/");
}

void read_word_images(const config& /*conf*/, spot_dataset& dataset, const std::string& path) {
    read_images(dataset.word_images, path + "/data/word_images_normalized/");
}

void read_word_images_gw(const config& conf, spot_dataset& dataset, const std::string& path) {
    if(conf.gray){
        read_images(dataset.word_images, path + "/data/word_gray/");
    } else if(conf.binary){
        read_images(dataset.word_images, path + "/data/word_binary/");
    } else {
        read_images(dataset.word_images, path + "/data/word_images_normalized/");
    }
}

void read_word_images_ak(const config& conf, spot_dataset& dataset, const std::string& path) {
    read_images_ak(conf, dataset.word_images, path + "/data/word_images_normalized", "test");
    read_images_ak(conf, dataset.word_images, path + "/data/word_images_normalized", "train");
}

void read_list(std::vector<std::string>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string image_name;
        stream >> image_name;

        if (!image_name.empty()) {
            list.push_back(image_name);
        }
    }
}

void read_keywords(std::vector<std::vector<std::string>>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string keywords;
        stream >> keywords;

        if (!keywords.empty()) {
            list.emplace_back();

            std::istringstream ss(keywords);
            std::string token;

            while (std::getline(ss, token, '-')) {
                list.back().push_back(token);
            }
        }
    }
}

void read_keywords_iam(std::vector<std::vector<std::string>>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string line_keywords;
        std::getline(stream, line_keywords);

        if(line_keywords.empty()){
            continue;
        }

        std::string keywords(line_keywords.begin(), line_keywords.begin() + line_keywords.find(' '));

        if(keywords.empty()){
            continue;
        }

        list.emplace_back();

        std::istringstream ss(keywords);
        std::string token;

        while (std::getline(ss, token, '-')) {
            list.back().push_back(token);
        }
    }
}

void load_sets_washington(spot_dataset& dataset, const std::string& path) {
    std::string file_path(path + "/sets");

    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 2) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        read_list(dataset.sets[file_name].test_set, full_name + "/test.txt");
        read_list(dataset.sets[file_name].train_set, full_name + "/train.txt");
        read_list(dataset.sets[file_name].validation_set, full_name + "/valid.txt");
        read_keywords(dataset.sets[file_name].keywords, full_name + "/keywords.txt");
    }
}

void load_sets_ak(spot_dataset& dataset, const std::string& path) {
    std::string set_name = "cv1";

    auto& test_set  = dataset.sets[set_name].test_set;
    auto& train_set = dataset.sets[set_name].train_set;
    auto& valid_set = dataset.sets[set_name].validation_set;

    for(auto& image : dataset.word_images){
        if(image.first.substr(0, 5) == "train"){
            train_set.push_back(image.first);
        } else if(image.first.substr(0, 4) == "test"){
            test_set.push_back(image.first);
        }
    }

    valid_set.clear(); // Nothing
}

void load_sets_parzival(spot_dataset& dataset, const std::string& path) {
    std::string full_name(path + "/sets1/");

    read_list(dataset.sets["cv1"].test_set, full_name + "/test.txt");
    read_list(dataset.sets["cv1"].train_set, full_name + "/train.txt");
    read_list(dataset.sets["cv1"].validation_set, full_name + "/valid.txt");
    read_keywords(dataset.sets["cv1"].keywords, full_name + "/keywords.txt");
}

void load_sets_iam(spot_dataset& dataset, const std::string& path) {
    std::string file_path(path + "/sets");

    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 2) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        read_list(dataset.sets[file_name].test_set, full_name + "/test.txt");
        read_list(dataset.sets[file_name].train_set, full_name + "/train.txt");
        read_list(dataset.sets[file_name].validation_set, full_name + "/valid.txt");
        read_keywords_iam(dataset.sets[file_name].keywords, full_name + "/keywords.txt");
    }
}

} //end of anonymous namespace

spot_dataset read_washington(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images_gw(conf, dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_washington(dataset, path);

    return dataset;
}

spot_dataset read_ak(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_labels_ak(dataset, path);
    read_word_images_ak(conf, dataset, path);

    // Now we need to filter the crap out of the data set

    auto it = dataset.word_images.begin();

    while (it != dataset.word_images.end()) {
        if (!dataset.word_labels.count({it->first.begin(), it->first.end() - 4})) {
            it = dataset.word_images.erase(it);
        } else {
            ++it;
        }
    }

    load_sets_ak(dataset, path);

    return dataset;
}

spot_dataset read_botany(const config& conf, const std::string& path) {
    // Seems the same crap
    return read_ak(conf, path);
}

spot_dataset read_manmatha(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_images(conf, dataset, path);

    load_sets_washington(dataset, path);

    return dataset;
}

spot_dataset read_parzival(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images(conf, dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_parzival(dataset, path);

    return dataset;
}

spot_dataset read_iam(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images(conf, dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_iam(dataset, path);

    return dataset;
}

std::vector<std::vector<std::string>> select_keywords(const config& conf, const spot_dataset& dataset, const spot_dataset_set& set, names train_word_names, names test_image_names, bool verbose) {
    std::vector<std::vector<std::string>> keywords;

    if(conf.ak || conf.botany){
        std::set<std::vector<std::string>> base_keywords;

        for (auto& labels : dataset.word_labels) {
            base_keywords.insert(labels.second);
        }

        for (auto& keyword : base_keywords) {
            bool found = false;

            for (auto& labels : dataset.word_labels) {
                if (keyword == labels.second && std::find(train_word_names.begin(), train_word_names.end(), labels.first) != train_word_names.end()) {
                    found = true;
                    break;
                }
            }

            if (found) {
                auto total_test = std::count_if(test_image_names.begin(), test_image_names.end(),
                                                [&dataset, &keyword](auto& i) {
                                                    std::string key{i.begin(), i.end() - 4};
                                                    return dataset.word_labels.at(key) == keyword;
                                                });

                if (total_test > 0) {
                    keywords.push_back(keyword);
                }
            }
        }
    } else {
        for (auto& keyword : set.keywords) {
            bool found = false;

            for (auto& labels : dataset.word_labels) {
                if (keyword == labels.second && std::find(train_word_names.begin(), train_word_names.end(), labels.first) != train_word_names.end()) {
                    found = true;
                    break;
                }
            }

            if (found) {
                auto total_test = std::count_if(test_image_names.begin(), test_image_names.end(),
                                                [&dataset, &keyword](auto& i) { return dataset.word_labels.at({i.begin(), i.end() - 4}) == keyword; });

                if (total_test > 0) {
                    keywords.push_back(keyword);
                }
            }
        }
    }

    if(verbose){
         std::cout << "Selected " << keywords.size() << " keyword out of " << set.keywords.size() << std::endl;
    }

    return keywords;
}
<commit_msg>More resizing<commit_after>//=======================================================================
// Copyright Baptiste Wicht 2015-2017.
// Distributed under the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <fstream>
#include <sstream>

#include <dirent.h>

#include "dataset.hpp"
#include "config.hpp"
#include "utils.hpp"

namespace {

void read_word_labels(spot_dataset& dataset, const std::string& path) {
    std::ifstream word_labels_stream(path + "/ground_truth/word_labels.txt");

    while (!word_labels_stream.eof()) {
        std::string image_name;
        word_labels_stream >> image_name;

        std::string label;
        word_labels_stream >> label;

        if (image_name.empty() || label.empty()) {
            continue;
        }

        std::istringstream ss(label);
        std::string token;

        while (std::getline(ss, token, '-')) {
            dataset.word_labels[image_name].push_back(token);
        }
    }
}

void read_word_labels_ak(spot_dataset& dataset, const std::string& path) {
    std::ifstream word_labels_stream(path + "/ground_truth/word_labels.txt");

    while (!word_labels_stream.eof()) {
        std::string image_name;
        word_labels_stream >> image_name;

        std::string label;
        word_labels_stream >> label;

        if (image_name.empty() || label.empty()) {
            continue;
        }

        // Get rid of AK crap
        if (label == "N/A" || label == "." || label == "," || label == "\"" || label == "-") {
            continue;
        }

        std::transform(label.begin(), label.end(), label.begin(), ::tolower);

        for(char c : label){
            dataset.word_labels[image_name].push_back(std::string() + c);
        }
    }
}

void read_line_transcriptions(spot_dataset& dataset, const std::string& path) {
    std::ifstream line_transcriptions_stream(path + "/ground_truth/transcription.txt");

    while (!line_transcriptions_stream.eof()) {
        std::string image_name;
        line_transcriptions_stream >> image_name;

        std::string label;
        line_transcriptions_stream >> label;

        if (image_name.empty() || label.empty()) {
            continue;
        }

        std::istringstream ss(label);
        std::string word_token;

        while (std::getline(ss, word_token, '|')) {
            std::vector<std::string> word_labels;

            std::istringstream ss2(word_token);
            std::string token;

            while (std::getline(ss2, token, '-')) {
                word_labels.push_back(token);
            }

            dataset.line_transcriptions[image_name].emplace_back(std::move(word_labels));
        }
    }
}

void read_images(std::unordered_map<std::string, cv::Mat>& map, const std::string& file_path) {
    std::cout << "Read images from '" << file_path << "'" << std::endl;

    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 3 || file_name.find(".png") != file_name.size() - 4) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        map[file_name] = cv::imread(full_name, CV_LOAD_IMAGE_ANYDEPTH);

        if (!map[file_name].data) {
            std::cout << "Impossible to read image " << full_name << std::endl;
        }
    }
}

void read_images_ak(const config& conf, std::unordered_map<std::string, cv::Mat>& map, const std::string& file_path, const std::string& sub) {
    std::cout << "Read images from '" << file_path << "/" << sub << "'" << std::endl;

    auto full_path = file_path + "/" + sub;

    struct dirent* entry;
    auto dir = opendir(full_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 3 || file_name.find(".png") != file_name.size() - 4) {
            continue;
        }

        std::string full_name(full_path + "/" + file_name);

        auto key = sub + "/" + file_name;

        cv::Mat base_image = cv::imread(full_name, CV_LOAD_IMAGE_ANYDEPTH);

        // Some methods need resizing
        if (conf.method == Method::Patches || conf.method == Method::Terasawa2009 || conf.method == Method::Rodriguez2008) {
            if (base_image.size().height == HEIGHT) {
                map[key] = base_image;
            } else {
                float ratio = float(HEIGHT) / base_image.size().height;

                cv::Mat scaled_normalized(
                    cv::Size(static_cast<size_t>(base_image.size().width * ratio), HEIGHT),
                    CV_8U);
                cv::resize(base_image, scaled_normalized, scaled_normalized.size(), cv::INTER_AREA);
                cv::adaptiveThreshold(scaled_normalized, map[key], 255, CV_ADAPTIVE_THRESH_MEAN_C, CV_THRESH_BINARY, 7, 2);
            }
        } else {
            map[key] = base_image;
        }

        if (!map[key].data) {
            std::cout << "Impossible to read image " << full_name << std::endl;
        }
    }
}

void read_line_images(spot_dataset& dataset, const std::string& path) {
    read_images(dataset.line_images, path + "/data/line_images_normalized/");
}

void read_word_images(const config& /*conf*/, spot_dataset& dataset, const std::string& path) {
    read_images(dataset.word_images, path + "/data/word_images_normalized/");
}

void read_word_images_gw(const config& conf, spot_dataset& dataset, const std::string& path) {
    if(conf.gray){
        read_images(dataset.word_images, path + "/data/word_gray/");
    } else if(conf.binary){
        read_images(dataset.word_images, path + "/data/word_binary/");
    } else {
        read_images(dataset.word_images, path + "/data/word_images_normalized/");
    }
}

void read_word_images_ak(const config& conf, spot_dataset& dataset, const std::string& path) {
    read_images_ak(conf, dataset.word_images, path + "/data/word_images_normalized", "test");
    read_images_ak(conf, dataset.word_images, path + "/data/word_images_normalized", "train");
}

void read_list(std::vector<std::string>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string image_name;
        stream >> image_name;

        if (!image_name.empty()) {
            list.push_back(image_name);
        }
    }
}

void read_keywords(std::vector<std::vector<std::string>>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string keywords;
        stream >> keywords;

        if (!keywords.empty()) {
            list.emplace_back();

            std::istringstream ss(keywords);
            std::string token;

            while (std::getline(ss, token, '-')) {
                list.back().push_back(token);
            }
        }
    }
}

void read_keywords_iam(std::vector<std::vector<std::string>>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string line_keywords;
        std::getline(stream, line_keywords);

        if(line_keywords.empty()){
            continue;
        }

        std::string keywords(line_keywords.begin(), line_keywords.begin() + line_keywords.find(' '));

        if(keywords.empty()){
            continue;
        }

        list.emplace_back();

        std::istringstream ss(keywords);
        std::string token;

        while (std::getline(ss, token, '-')) {
            list.back().push_back(token);
        }
    }
}

void load_sets_washington(spot_dataset& dataset, const std::string& path) {
    std::string file_path(path + "/sets");

    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 2) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        read_list(dataset.sets[file_name].test_set, full_name + "/test.txt");
        read_list(dataset.sets[file_name].train_set, full_name + "/train.txt");
        read_list(dataset.sets[file_name].validation_set, full_name + "/valid.txt");
        read_keywords(dataset.sets[file_name].keywords, full_name + "/keywords.txt");
    }
}

void load_sets_ak(spot_dataset& dataset, const std::string& path) {
    std::string set_name = "cv1";

    auto& test_set  = dataset.sets[set_name].test_set;
    auto& train_set = dataset.sets[set_name].train_set;
    auto& valid_set = dataset.sets[set_name].validation_set;

    for(auto& image : dataset.word_images){
        if(image.first.substr(0, 5) == "train"){
            train_set.push_back(image.first);
        } else if(image.first.substr(0, 4) == "test"){
            test_set.push_back(image.first);
        }
    }

    valid_set.clear(); // Nothing
}

void load_sets_parzival(spot_dataset& dataset, const std::string& path) {
    std::string full_name(path + "/sets1/");

    read_list(dataset.sets["cv1"].test_set, full_name + "/test.txt");
    read_list(dataset.sets["cv1"].train_set, full_name + "/train.txt");
    read_list(dataset.sets["cv1"].validation_set, full_name + "/valid.txt");
    read_keywords(dataset.sets["cv1"].keywords, full_name + "/keywords.txt");
}

void load_sets_iam(spot_dataset& dataset, const std::string& path) {
    std::string file_path(path + "/sets");

    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 2) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        read_list(dataset.sets[file_name].test_set, full_name + "/test.txt");
        read_list(dataset.sets[file_name].train_set, full_name + "/train.txt");
        read_list(dataset.sets[file_name].validation_set, full_name + "/valid.txt");
        read_keywords_iam(dataset.sets[file_name].keywords, full_name + "/keywords.txt");
    }
}

} //end of anonymous namespace

spot_dataset read_washington(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images_gw(conf, dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_washington(dataset, path);

    return dataset;
}

spot_dataset read_ak(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_labels_ak(dataset, path);
    read_word_images_ak(conf, dataset, path);

    // Now we need to filter the crap out of the data set

    auto it = dataset.word_images.begin();

    while (it != dataset.word_images.end()) {
        if (!dataset.word_labels.count({it->first.begin(), it->first.end() - 4})) {
            it = dataset.word_images.erase(it);
        } else {
            ++it;
        }
    }

    load_sets_ak(dataset, path);

    return dataset;
}

spot_dataset read_botany(const config& conf, const std::string& path) {
    // Seems the same crap
    return read_ak(conf, path);
}

spot_dataset read_manmatha(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_images(conf, dataset, path);

    load_sets_washington(dataset, path);

    return dataset;
}

spot_dataset read_parzival(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images(conf, dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_parzival(dataset, path);

    return dataset;
}

spot_dataset read_iam(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images(conf, dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_iam(dataset, path);

    return dataset;
}

std::vector<std::vector<std::string>> select_keywords(const config& conf, const spot_dataset& dataset, const spot_dataset_set& set, names train_word_names, names test_image_names, bool verbose) {
    std::vector<std::vector<std::string>> keywords;

    if(conf.ak || conf.botany){
        std::set<std::vector<std::string>> base_keywords;

        for (auto& labels : dataset.word_labels) {
            base_keywords.insert(labels.second);
        }

        for (auto& keyword : base_keywords) {
            bool found = false;

            for (auto& labels : dataset.word_labels) {
                if (keyword == labels.second && std::find(train_word_names.begin(), train_word_names.end(), labels.first) != train_word_names.end()) {
                    found = true;
                    break;
                }
            }

            if (found) {
                auto total_test = std::count_if(test_image_names.begin(), test_image_names.end(),
                                                [&dataset, &keyword](auto& i) {
                                                    std::string key{i.begin(), i.end() - 4};
                                                    return dataset.word_labels.at(key) == keyword;
                                                });

                if (total_test > 0) {
                    keywords.push_back(keyword);
                }
            }
        }
    } else {
        for (auto& keyword : set.keywords) {
            bool found = false;

            for (auto& labels : dataset.word_labels) {
                if (keyword == labels.second && std::find(train_word_names.begin(), train_word_names.end(), labels.first) != train_word_names.end()) {
                    found = true;
                    break;
                }
            }

            if (found) {
                auto total_test = std::count_if(test_image_names.begin(), test_image_names.end(),
                                                [&dataset, &keyword](auto& i) { return dataset.word_labels.at({i.begin(), i.end() - 4}) == keyword; });

                if (total_test > 0) {
                    keywords.push_back(keyword);
                }
            }
        }
    }

    if(verbose){
         std::cout << "Selected " << keywords.size() << " keyword out of " << set.keywords.size() << std::endl;
    }

    return keywords;
}
<|endoftext|>


<commit_before>#include "Kdtree.h"
#include <cstdio>
#include <algorithm>
#include <numeric>
#include <cmath>

namespace P_RVD{
	// Eww global. Need this for the sort function
	// A pointer to the class itself, allows the sort function to determine the splitting axis to sort by
	static KDtree *myself = NULL;

	KDtree::KDtree()
	{
		myself = this;
		m_id = 0;
	}

	KDtree::~KDtree()
	{
		// Delete all the ndoes
		std::vector <KDNode*> to_visit;

		to_visit.push_back(m_root);

		while (to_visit.size()) {
			std::vector <KDNode*> next_search;

			while (to_visit.size()) {
				KDNode *cur = to_visit.back();
				to_visit.pop_back();

				if (cur->left)
					next_search.push_back(cur->left);

				if (cur->right)
					next_search.push_back(cur->right);

				delete cur;
			}

			to_visit = next_search;
		}

		m_root = NULL;
	}

	void KDtree::Create_kdtree(std::vector<Kd_tree_point> &pts, int max_levels)
	{
		m_pts = &pts;
		m_levels = max_levels;

		m_root = new KDNode();
		m_root->id = m_id++;
		m_root->index.resize(pts.size());

		//init the index
		for (unsigned int i = 0; i < pts.size(); ++i)
		{
			m_root->index[i] = i;
		}

		std::vector<KDNode*> to_visit;
		to_visit.push_back(m_root);

		while (to_visit.size())
		{
			std::vector<KDNode*> next_search;
			while (to_visit.size())
			{
				KDNode *node = to_visit.back();
				to_visit.pop_back();;

				if (node->level < max_levels)
				{
					//if current node's index size > 1
					//it has more than 1 point in this space
					//So split it
					if (node->index.size() > 1)
					{
						KDNode *left = new KDNode();
						KDNode *right = new KDNode();

						left->id = m_id++;
						right->id = m_id++;

						Split(node, left, right);

						//Claer current index
						{
							std::vector<int> dummy;
							node->index.swap(dummy);
						}

						node->left = left;
						node->right = right;

						node->_left = left->id;
						node->_right = right->id;

						if (left->index.size())
							next_search.push_back(left);

						if (right->index.size())
							next_search.push_back(right);
					}
				}
			}
			to_visit = next_search;
		}
	}

	/*
		compare the data of 2 points with the index
	*/
	bool KDtree::SortPoints(const int a, const int b)
	{
		std::vector<Kd_tree_point> &pts = *myself->m_pts;

		return pts[a].coords[myself->m_current_axis] < pts[b].coords[myself->m_current_axis];
	}

	/*
		Split the current node into 2 part: left and right

		we assume that the left and right nodes are already created
	*/
	void KDtree::Split(KDNode *cur, KDNode *left, KDNode *right)
	{
		std::vector<Kd_tree_point> &pts = *m_pts;
		m_current_axis = cur->level % KDTREE_DIM;

		//sort the points by the axis value
		std::sort(cur->index.begin(), cur->index.end(), KDtree::SortPoints);

		int mid = cur->index[cur->index.size() / 2];
		cur->split_value = pts[mid].coords[m_current_axis];

		left->parent = cur;
		right->parent = cur;

		left->level = cur->level + 1;
		right->level = cur->level + 1;

		left->_parent = cur->id;
		right->_parent = cur->id;

		for (unsigned int i = 0; i < cur->index.size(); ++i)
		{
			int idx = cur->index[i];
			if (pts[idx].coords[m_current_axis] < cur->split_value)
				left->index.push_back(idx);
			else
				right->index.push_back(idx);
		}
	}

	/*
		search the best node
		return the index and distance square of the closed point in this node
	*/
	void KDtree::SearchAtNode(KDNode *cur, const Kd_tree_point &query, int *ret_index, double *ret_dist, KDNode **ret_node)
	{
		int best_idx = 0;
		// if the range the coords is larger than 100,best_dist should be adjusted
		double best_dist_Squa = 100.0;

		std::vector<Kd_tree_point> &pts = *m_pts;

		//First pass
		while (true)
		{
			int split_axis = cur->level % KDTREE_DIM;
			m_cmps++;

			if (cur->left == NULL)
			{
				*ret_node = cur;

				for (unsigned int i = 0; i < cur->index.size(); ++i)
				{
					m_cmps++;

					int idx = cur->index[i];

					double distSqua = DistanceSquare(query, pts[idx]);
					if (distSqua < best_dist_Squa)
					{
						best_dist_Squa = distSqua;
						best_idx = idx;
					}
				}
				break;
			}
			else if (query.coords[split_axis] < cur->split_value)
			{
				cur = cur->left;
			}
			else
			{
				cur = cur->right;
			}
		}
		*ret_index = best_idx;
		*ret_dist = best_dist_Squa;
	}

	void KDtree::SearchAtNodeRange(KDNode *cur, const Kd_tree_point &query, double range, int *ret_index, double *ret_dist)
	{
		int best_idx = 0;
		double best_dist = 100.0;

		std::vector<Kd_tree_point> &pts = *m_pts;
		std::vector<KDNode* > to_visit;

		to_visit.push_back(cur);
		while (to_visit.size() > 0)
		{
			std::vector<KDNode*> next_search;
			while (to_visit.size())
			{
				cur = to_visit.back();
				to_visit.pop_back();

				int split_axis = cur->level % KDTREE_DIM;

				if (cur->left == NULL)
				{
					for (unsigned int i = 0; i < cur->index.size(); i++) {
						m_cmps++;

						int idx = cur->index[i];
						double d = DistanceSquare(query, pts[idx]);

						if (d < best_dist) {
							best_dist = d;
							best_idx = idx;
						}
					}
				}
				else
				{
					double d = query.coords[split_axis] - cur->split_value;

					// There are 3 possible scenarios
					// The hypercircle only intersects the left region
					// The hypercircle only intersects the right region
					// The hypercricle intersects both

					m_cmps++;

					if (fabs(d) > range) {
						if (d < 0)
							next_search.push_back(cur->left);
						else
							next_search.push_back(cur->right);
					}
					else {
						next_search.push_back(cur->left);
						next_search.push_back(cur->right);
					}
				}
			}
		}

	}

	void KDtree::Search(const Kd_tree_point &query, int *ret_index, double *ret_dist)
	{
		//First pass
		//Find the first closest node, this will be the upper bound for the next searches
		std::vector<Kd_tree_point> &pts = *m_pts;
		KDNode *best_node = NULL;
		int best_idx = 0;
		double best_dist = 100.0;
		double radius = 0.0;
		m_cmps = 0;

		SearchAtNode(m_root, query, &best_idx, &best_dist, &best_node);
		radius = sqrt(best_dist);

		//Second pss
		//Find ohter possible candidates
		KDNode *cur = best_node;
		while (cur->parent != NULL)
		{
			//Go up
			KDNode *parent = cur->parent;
			int split_axis = (parent->level) % KDTREE_DIM;

			//Search the other node
			int temp_idx;
			double temp_dist = 100.0;
			KDNode *temp_node;
			KDNode *search_node = NULL;

			if (fabs(parent->split_value - query.coords[split_axis]) <= radius)
			{
				//Search opposite node
				if (parent->left != cur)
					SearchAtNodeRange(parent->left, query, radius, &temp_idx, &temp_dist);
				else
					SearchAtNodeRange(parent->right, query, radius, &temp_idx, &temp_dist);
			}

			if (temp_dist < best_dist)
			{
				best_idx = temp_idx;
				best_dist = temp_dist;
			}
			cur = parent;
		}
		*ret_index = best_idx;
		*ret_dist = best_dist;
	}

	inline double KDtree::DistanceSquare(const Kd_tree_point &a, const Kd_tree_point &b) const
	{
		double dist = 0.0;

		for (int i = 0; i < KDTREE_DIM; ++i)
		{
			double d = a.coords[i] - b.coords[i];
			dist += d * d;
		}
		return dist;
	}
}<commit_msg>modify for knn<commit_after>#include "Kdtree.h"
#include <cstdio>
#include <algorithm>
#include <numeric>
#include <cmath>

namespace P_RVD{
	// Eww global. Need this for the sort function
	// A pointer to the class itself, allows the sort function to determine the splitting axis to sort by
	static KDtree *myself = NULL;

	KDtree::KDtree()
	{
		myself = this;
		m_id = 0;
	}

	KDtree::~KDtree()
	{
		// Delete all the ndoes
		std::vector <KDNode*> to_visit;

		to_visit.push_back(m_root);

		while (to_visit.size()) {
			std::vector <KDNode*> next_search;

			while (to_visit.size()) {
				KDNode *cur = to_visit.back();
				to_visit.pop_back();

				if (cur->left)
					next_search.push_back(cur->left);

				if (cur->right)
					next_search.push_back(cur->right);

				delete cur;
			}

			to_visit = next_search;
		}

		m_root = NULL;
	}

	void KDtree::Create_kdtree(std::vector<Kd_tree_point> &pts, int max_levels)
	{
		m_pts = &pts;
		m_levels = max_levels;

		m_root = new KDNode();
		m_root->id = m_id++;
		m_root->index.resize(pts.size());

		//init the index
		for (unsigned int i = 0; i < pts.size(); ++i)
		{
			m_root->index[i] = i;
		}

		std::vector<KDNode*> to_visit;
		to_visit.push_back(m_root);

		while (to_visit.size())
		{
			std::vector<KDNode*> next_search;
			while (to_visit.size())
			{
				KDNode *node = to_visit.back();
				to_visit.pop_back();;

				if (node->level < max_levels)
				{
					//if current node's index size > 1
					//it has more than 1 point in this space
					//So split it
					if (node->index.size() > 1)
					{
						KDNode *left = new KDNode();
						KDNode *right = new KDNode();

						left->id = m_id++;
						right->id = m_id++;

						Split(node, left, right);

						//Claer current index
						{
							//std::vector<int> dummy;
							//node->index.swap(dummy);
						}

						node->left = left;
						node->right = right;

						node->_left = left->id;
						node->_right = right->id;

						if (left->index.size())
							next_search.push_back(left);

						if (right->index.size())
							next_search.push_back(right);
					}
				}
			}
			to_visit = next_search;
		}
	}

	/*
		compare the data of 2 points with the index
	*/
	bool KDtree::SortPoints(const int a, const int b)
	{
		std::vector<Kd_tree_point> &pts = *myself->m_pts;

		return pts[a].coords[myself->m_current_axis] < pts[b].coords[myself->m_current_axis];
	}

	/*
		Split the current node into 2 part: left and right

		we assume that the left and right nodes are already created
	*/
	void KDtree::Split(KDNode *cur, KDNode *left, KDNode *right)
	{
		std::vector<Kd_tree_point> &pts = *m_pts;
		m_current_axis = cur->level % KDTREE_DIM;

		//sort the points by the axis value
		std::sort(cur->index.begin(), cur->index.end(), KDtree::SortPoints);

		int mid = cur->index[cur->index.size() / 2];
		cur->split_value = pts[mid].coords[m_current_axis];

		left->parent = cur;
		right->parent = cur;

		left->level = cur->level + 1;
		right->level = cur->level + 1;

		left->_parent = cur->id;
		right->_parent = cur->id;

		for (unsigned int i = 0; i < cur->index.size(); ++i)
		{
			int idx = cur->index[i];
			if (pts[idx].coords[m_current_axis] < cur->split_value)
				left->index.push_back(idx);
			else
				right->index.push_back(idx);
		}
	}

	/*
		search the best node
		return the index and distance square of the closed point in this node
	*/
	void KDtree::SearchAtNode(KDNode *cur, const Kd_tree_point &query, int *ret_index, double *ret_dist, KDNode **ret_node)
	{
		int best_idx = 0;
		// if the range the coords is larger than 100,best_dist should be adjusted
		double best_dist_Squa = 100.0;

		std::vector<Kd_tree_point> &pts = *m_pts;

		//First pass
		while (true)
		{
			int split_axis = cur->level % KDTREE_DIM;
			m_cmps++;

			if (cur->left == NULL)
			{
				*ret_node = cur;

				for (unsigned int i = 0; i < cur->index.size(); ++i)
				{
					m_cmps++;

					int idx = cur->index[i];

					double distSqua = DistanceSquare(query, pts[idx]);
					if (distSqua < best_dist_Squa)
					{
						best_dist_Squa = distSqua;
						best_idx = idx;
					}
				}
				break;
			}
			else if (query.coords[split_axis] < cur->split_value)
			{
				cur = cur->left;
			}
			else
			{
				cur = cur->right;
			}
		}
		*ret_index = best_idx;
		*ret_dist = best_dist_Squa;
	}

	void KDtree::SearchAtNodeRange(KDNode *cur, const Kd_tree_point &query, double range, int *ret_index, double *ret_dist)
	{
		int best_idx = 0;
		double best_dist = 100.0;

		std::vector<Kd_tree_point> &pts = *m_pts;
		std::vector<KDNode* > to_visit;

		to_visit.push_back(cur);
		while (to_visit.size() > 0)
		{
			std::vector<KDNode*> next_search;
			while (to_visit.size())
			{
				cur = to_visit.back();
				to_visit.pop_back();

				int split_axis = cur->level % KDTREE_DIM;

				if (cur->left == NULL)
				{
					for (unsigned int i = 0; i < cur->index.size(); i++) {
						m_cmps++;

						int idx = cur->index[i];
						double d = DistanceSquare(query, pts[idx]);

						if (d < best_dist) {
							best_dist = d;
							best_idx = idx;
						}
					}
				}
				else
				{
					double d = query.coords[split_axis] - cur->split_value;

					// There are 3 possible scenarios
					// The hypercircle only intersects the left region
					// The hypercircle only intersects the right region
					// The hypercricle intersects both

					m_cmps++;

					if (fabs(d) > range) {
						if (d < 0)
							next_search.push_back(cur->left);
						else
							next_search.push_back(cur->right);
					}
					else {
						next_search.push_back(cur->left);
						next_search.push_back(cur->right);
					}
				}
			}
		}

	}

	void KDtree::Search(const Kd_tree_point &query, int *ret_index, double *ret_dist)
	{
		//First pass
		//Find the first closest node, this will be the upper bound for the next searches
		std::vector<Kd_tree_point> &pts = *m_pts;
		KDNode *best_node = NULL;
		int best_idx = 0;
		double best_dist = 100.0;
		double radius = 0.0;
		m_cmps = 0;

		SearchAtNode(m_root, query, &best_idx, &best_dist, &best_node);
		radius = sqrt(best_dist);

		//Second pss
		//Find ohter possible candidates
		KDNode *cur = best_node;
		while (cur->parent != NULL)
		{
			//Go up
			KDNode *parent = cur->parent;
			int split_axis = (parent->level) % KDTREE_DIM;

			//Search the other node
			int temp_idx;
			double temp_dist = 100.0;
			KDNode *temp_node;
			KDNode *search_node = NULL;

			if (fabs(parent->split_value - query.coords[split_axis]) <= radius)
			{
				//Search opposite node
				if (parent->left != cur)
					SearchAtNodeRange(parent->left, query, radius, &temp_idx, &temp_dist);
				else
					SearchAtNodeRange(parent->right, query, radius, &temp_idx, &temp_dist);
			}

			if (temp_dist < best_dist)
			{
				best_idx = temp_idx;
				best_dist = temp_dist;
			}
			cur = parent;
		}
		*ret_index = best_idx;
		*ret_dist = best_dist;
	}

	inline double KDtree::DistanceSquare(const Kd_tree_point &a, const Kd_tree_point &b) const
	{
		double dist = 0.0;

		for (int i = 0; i < KDTREE_DIM; ++i)
		{
			double d = a.coords[i] - b.coords[i];
			dist += d * d;
		}
		return dist;
	}
}<|endoftext|>


<commit_before>//=======================================================================
// Copyright Baptiste Wicht 2015.
// Distributed under the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <fstream>
#include <sstream>

#include <dirent.h>

#include "dataset.hpp"
#include "config.hpp"

namespace {

void read_word_labels(spot_dataset& dataset, const std::string& path) {
    std::ifstream word_labels_stream(path + "/ground_truth/word_labels.txt");

    while (!word_labels_stream.eof()) {
        std::string image_name;
        word_labels_stream >> image_name;

        std::string label;
        word_labels_stream >> label;

        if (image_name.empty() || label.empty()) {
            continue;
        }

        std::istringstream ss(label);
        std::string token;

        while (std::getline(ss, token, '-')) {
            dataset.word_labels[image_name].push_back(token);
        }
    }
}

void read_line_transcriptions(spot_dataset& dataset, const std::string& path) {
    std::ifstream line_transcriptions_stream(path + "/ground_truth/transcription.txt");

    while (!line_transcriptions_stream.eof()) {
        std::string image_name;
        line_transcriptions_stream >> image_name;

        std::string label;
        line_transcriptions_stream >> label;

        if (image_name.empty() || label.empty()) {
            continue;
        }

        std::istringstream ss(label);
        std::string word_token;

        while (std::getline(ss, word_token, '|')) {
            std::vector<std::string> word_labels;

            std::istringstream ss2(word_token);
            std::string token;

            while (std::getline(ss2, token, '-')) {
                word_labels.push_back(token);
            }

            dataset.line_transcriptions[image_name].emplace_back(std::move(word_labels));
        }
    }
}

void read_images(std::unordered_map<std::string, cv::Mat>& map, const std::string& file_path) {
    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 3 || file_name.find(".png") != file_name.size() - 4) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        map[file_name] = cv::imread(full_name, CV_LOAD_IMAGE_ANYDEPTH);

        if (!map[file_name].data) {
            std::cout << "Impossible to read image " << full_name << std::endl;
        }
    }
}

void read_line_images(spot_dataset& dataset, const std::string& path) {
    read_images(dataset.line_images, path + "/data/line_images_normalized/");
}

void read_word_images(spot_dataset& dataset, const std::string& path) {
    read_images(dataset.word_images, path + "/data/word_images_normalized/");
}

void read_list(std::vector<std::string>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string image_name;
        stream >> image_name;

        if (!image_name.empty()) {
            list.push_back(image_name);
        }
    }
}

void read_keywords(std::vector<std::vector<std::string>>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string keywords;
        stream >> keywords;

        if (!keywords.empty()) {
            list.emplace_back();

            std::istringstream ss(keywords);
            std::string token;

            while (std::getline(ss, token, '-')) {
                list.back().push_back(token);
            }
        }
    }
}

void read_keywords_iam(std::vector<std::vector<std::string>>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string line_keywords;
        std::getline(stream, line_keywords);

        if(line_keywords.empty()){
            continue;
        }

        std::string keywords(line_keywords.begin(), line_keywords.begin() + line_keywords.find(' '));

        if(keywords.empty()){
            continue;
        }

        list.emplace_back();

        std::istringstream ss(keywords);
        std::string token;

        while (std::getline(ss, token, '-')) {
            list.back().push_back(token);
        }
    }
}

void load_sets_washington(spot_dataset& dataset, const std::string& path) {
    std::string file_path(path + "/sets");

    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 2) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        read_list(dataset.sets[file_name].test_set, full_name + "/test.txt");
        read_list(dataset.sets[file_name].train_set, full_name + "/train.txt");
        read_list(dataset.sets[file_name].validation_set, full_name + "/valid.txt");
        read_keywords(dataset.sets[file_name].keywords, full_name + "/keywords.txt");
    }
}

void load_sets_parzival(spot_dataset& dataset, const std::string& path) {
    std::string full_name(path + "/sets1/");

    read_list(dataset.sets["cv1"].test_set, full_name + "/test.txt");
    read_list(dataset.sets["cv1"].train_set, full_name + "/train.txt");
    read_list(dataset.sets["cv1"].validation_set, full_name + "/valid.txt");
    read_keywords(dataset.sets["cv1"].keywords, full_name + "/keywords.txt");
}

void load_sets_iam(spot_dataset& dataset, const std::string& path) {
    std::string full_name(path + "/sets/");

    read_list(dataset.sets["cv1"].test_set, full_name + "/test.txt");
    read_list(dataset.sets["cv1"].train_set, full_name + "/train.txt");
    read_list(dataset.sets["cv1"].validation_set, full_name + "/valid.txt");
    read_keywords_iam(dataset.sets["cv1"].keywords, full_name + "/keywords.txt");
}

} //end of anonymous namespace

spot_dataset read_washington(const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images(dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_washington(dataset, path);

    return dataset;
}

spot_dataset read_parzival(const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images(dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_parzival(dataset, path);

    return dataset;
}

spot_dataset read_iam(const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images(dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_iam(dataset, path);

    return dataset;
}

std::vector<std::vector<std::string>> select_keywords(const spot_dataset& dataset, const spot_dataset_set& set, names train_word_names, names test_image_names) {
    std::vector<std::vector<std::string>> keywords;

    for (auto& keyword : set.keywords) {
        bool found = false;

        for (auto& labels : dataset.word_labels) {
            if (keyword == labels.second && std::find(train_word_names.begin(), train_word_names.end(), labels.first) != train_word_names.end()) {
                found = true;
                break;
            }
        }

        if (found) {
            auto total_test = std::count_if(test_image_names.begin(), test_image_names.end(),
                                            [&dataset, &keyword](auto& i) { return dataset.word_labels.at({i.begin(), i.end() - 4}) == keyword; });

            if (total_test > 0) {
                keywords.push_back(keyword);
            }
        }
    }

    std::cout << "Selected " << keywords.size() << " keyword out of " << set.keywords.size() << std::endl;

    return keywords;
}
<commit_msg>Review iam sets<commit_after>//=======================================================================
// Copyright Baptiste Wicht 2015.
// Distributed under the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <fstream>
#include <sstream>

#include <dirent.h>

#include "dataset.hpp"
#include "config.hpp"

namespace {

void read_word_labels(spot_dataset& dataset, const std::string& path) {
    std::ifstream word_labels_stream(path + "/ground_truth/word_labels.txt");

    while (!word_labels_stream.eof()) {
        std::string image_name;
        word_labels_stream >> image_name;

        std::string label;
        word_labels_stream >> label;

        if (image_name.empty() || label.empty()) {
            continue;
        }

        std::istringstream ss(label);
        std::string token;

        while (std::getline(ss, token, '-')) {
            dataset.word_labels[image_name].push_back(token);
        }
    }
}

void read_line_transcriptions(spot_dataset& dataset, const std::string& path) {
    std::ifstream line_transcriptions_stream(path + "/ground_truth/transcription.txt");

    while (!line_transcriptions_stream.eof()) {
        std::string image_name;
        line_transcriptions_stream >> image_name;

        std::string label;
        line_transcriptions_stream >> label;

        if (image_name.empty() || label.empty()) {
            continue;
        }

        std::istringstream ss(label);
        std::string word_token;

        while (std::getline(ss, word_token, '|')) {
            std::vector<std::string> word_labels;

            std::istringstream ss2(word_token);
            std::string token;

            while (std::getline(ss2, token, '-')) {
                word_labels.push_back(token);
            }

            dataset.line_transcriptions[image_name].emplace_back(std::move(word_labels));
        }
    }
}

void read_images(std::unordered_map<std::string, cv::Mat>& map, const std::string& file_path) {
    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 3 || file_name.find(".png") != file_name.size() - 4) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        map[file_name] = cv::imread(full_name, CV_LOAD_IMAGE_ANYDEPTH);

        if (!map[file_name].data) {
            std::cout << "Impossible to read image " << full_name << std::endl;
        }
    }
}

void read_line_images(spot_dataset& dataset, const std::string& path) {
    read_images(dataset.line_images, path + "/data/line_images_normalized/");
}

void read_word_images(spot_dataset& dataset, const std::string& path) {
    read_images(dataset.word_images, path + "/data/word_images_normalized/");
}

void read_list(std::vector<std::string>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string image_name;
        stream >> image_name;

        if (!image_name.empty()) {
            list.push_back(image_name);
        }
    }
}

void read_keywords(std::vector<std::vector<std::string>>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string keywords;
        stream >> keywords;

        if (!keywords.empty()) {
            list.emplace_back();

            std::istringstream ss(keywords);
            std::string token;

            while (std::getline(ss, token, '-')) {
                list.back().push_back(token);
            }
        }
    }
}

void read_keywords_iam(std::vector<std::vector<std::string>>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string line_keywords;
        std::getline(stream, line_keywords);

        if(line_keywords.empty()){
            continue;
        }

        std::string keywords(line_keywords.begin(), line_keywords.begin() + line_keywords.find(' '));

        if(keywords.empty()){
            continue;
        }

        list.emplace_back();

        std::istringstream ss(keywords);
        std::string token;

        while (std::getline(ss, token, '-')) {
            list.back().push_back(token);
        }
    }
}

void load_sets_washington(spot_dataset& dataset, const std::string& path) {
    std::string file_path(path + "/sets");

    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 2) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        read_list(dataset.sets[file_name].test_set, full_name + "/test.txt");
        read_list(dataset.sets[file_name].train_set, full_name + "/train.txt");
        read_list(dataset.sets[file_name].validation_set, full_name + "/valid.txt");
        read_keywords(dataset.sets[file_name].keywords, full_name + "/keywords.txt");
    }
}

void load_sets_parzival(spot_dataset& dataset, const std::string& path) {
    std::string full_name(path + "/sets1/");

    read_list(dataset.sets["cv1"].test_set, full_name + "/test.txt");
    read_list(dataset.sets["cv1"].train_set, full_name + "/train.txt");
    read_list(dataset.sets["cv1"].validation_set, full_name + "/valid.txt");
    read_keywords(dataset.sets["cv1"].keywords, full_name + "/keywords.txt");
}

void load_sets_iam(spot_dataset& dataset, const std::string& path) {
    std::string file_path(path + "/sets");

    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 2) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        read_list(dataset.sets[file_name].test_set, full_name + "/test.txt");
        read_list(dataset.sets[file_name].train_set, full_name + "/train.txt");
        read_list(dataset.sets[file_name].validation_set, full_name + "/valid.txt");
        read_keywords_iam(dataset.sets[file_name].keywords, full_name + "/keywords.txt");
    }
}

} //end of anonymous namespace

spot_dataset read_washington(const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images(dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_washington(dataset, path);

    return dataset;
}

spot_dataset read_parzival(const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images(dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_parzival(dataset, path);

    return dataset;
}

spot_dataset read_iam(const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images(dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_iam(dataset, path);

    return dataset;
}

std::vector<std::vector<std::string>> select_keywords(const spot_dataset& dataset, const spot_dataset_set& set, names train_word_names, names test_image_names) {
    std::vector<std::vector<std::string>> keywords;

    for (auto& keyword : set.keywords) {
        bool found = false;

        for (auto& labels : dataset.word_labels) {
            if (keyword == labels.second && std::find(train_word_names.begin(), train_word_names.end(), labels.first) != train_word_names.end()) {
                found = true;
                break;
            }
        }

        if (found) {
            auto total_test = std::count_if(test_image_names.begin(), test_image_names.end(),
                                            [&dataset, &keyword](auto& i) { return dataset.word_labels.at({i.begin(), i.end() - 4}) == keyword; });

            if (total_test > 0) {
                keywords.push_back(keyword);
            }
        }
    }

    std::cout << "Selected " << keywords.size() << " keyword out of " << set.keywords.size() << std::endl;

    return keywords;
}
<|endoftext|>


<commit_before>#include <sched.h>
#include <linux/unistd.h>
#include <sys/types.h>
#include <sys/syscall.h>
#include <unistd.h>

#include "core_manager.h"
#include "core.h"
#include "network.h"
#include "ocache.h"
#include "config.h"
#include "packetize.h"
#include "message_types.h"

#include "log.h"

using namespace std;

CoreManager::CoreManager()
      :
      tid_to_core_map(3*Config::getSingleton()->getNumLocalCores()),
      tid_to_core_index_map(3*Config::getSingleton()->getNumLocalCores()),
      simthread_tid_to_core_map(3*Config::getSingleton()->getNumLocalCores())
{
   LOG_PRINT("Starting CoreManager Constructor.");

   UInt32 num_local_cores = Config::getSingleton()->getNumLocalCores();

   tid_map = new UInt32 [num_local_cores];
   core_to_simthread_tid_map = new UInt32 [num_local_cores];

   UInt32 proc_id = Config::getSingleton()->getCurrentProcessNum();
   const Config::CoreList &local_cores = Config::getSingleton()->getCoreListForProcess(proc_id);

   for (UInt32 i = 0; i < num_local_cores; i++)
   {
      tid_map[i] = UINT_MAX;
      core_to_simthread_tid_map[i] = UINT_MAX;
      m_cores.push_back(new Core(local_cores[i]));
   }

   LOG_PRINT("Finished CoreManager Constructor.");
}

CoreManager::~CoreManager()
{
   for (std::vector<Core *>::iterator i = m_cores.begin(); i != m_cores.end(); i++)
      delete *i;

   delete [] core_to_simthread_tid_map;
   delete [] tid_map;
}

void CoreManager::initializeCommId(SInt32 comm_id)
{
   UInt32 tid = getCurrentTID();
   pair<bool, UInt64> e = tid_to_core_map.find(tid);

   LOG_ASSERT_ERROR(e.first, "initializeCommId: Called without binding thread to a core.");
   core_id_t core_id = e.second;

   UnstructuredBuffer send_buff;
   send_buff << (SInt32)LCP_MESSAGE_COMMID_UPDATE << comm_id << core_id;

   LOG_PRINT("Initializing comm_id: %d to core_id: %d", comm_id, core_id);

   // Broadcast this update to other processes

   e = tid_to_core_index_map.find(tid);
   LOG_ASSERT_ERROR(e.first, "initializeCommId: tid mapped to core, but not to an index?");
   UInt32 idx = (UInt32) e.second;

   LOG_ASSERT_ERROR(idx < Config::getSingleton()->getNumLocalCores(), "CoreManager got and index [%d] out of range (0-%d).",
         idx, Config::getSingleton()->getNumLocalCores());

   Network *network = m_cores[idx]->getNetwork();
   Transport::Node *transport = Transport::getSingleton()->getGlobalNode();
   UInt32 num_procs = Config::getSingleton()->getProcessCount();

   for (UInt32 i = 0; i < num_procs; i++)
   {
      transport->globalSend(i,
                            send_buff.getBuffer(),
                            send_buff.size());
   }

   LOG_PRINT("Waiting for replies from LCPs.");

   for (UInt32 i = 0; i < num_procs; i++)
   {
      network->netRecvType(LCP_COMM_ID_UPDATE_REPLY);
      LOG_PRINT("Received reply from proc: %d", i);
   }

   LOG_PRINT("Finished.");
}

void CoreManager::initializeThread()
{
   ScopedLock scoped_maps_lock(m_maps_lock);
   UInt32 tid = getCurrentTID();
   pair<bool, UInt64> e = tid_to_core_map.find(tid);

   LOG_ASSERT_WARNING(e.first == false, "Thread: %d already mapped to core: %lld", tid, e.second);

   const Config::CoreList &core_list = Config::getSingleton()->getCoreListForProcess(Config::getSingleton()->getCurrentProcessNum());

   for (UInt32 i = 0; i < Config::getSingleton()->getNumLocalCores(); i++)
   {
      if (tid_map[i] == UINT_MAX)
      {
         core_id_t core_id = core_list[i];
         tid_map[i] = tid;
         tid_to_core_index_map.insert(tid, i);
         tid_to_core_map.insert(tid, core_id);
         LOG_PRINT("Initialize thread : index %d mapped to: thread %d, core_id: %d", i, tid, core_id);
         return;
      }
      else
      {
         // LOG_PRINT("%d/%d already mapped to: %d", i, Config::getSingleton()->getNumLocalCores(), tid_map[i]);
      }
   }

   LOG_PRINT_ERROR("initializeThread - No free cores out of %d total.", Config::getSingleton()->getNumLocalCores());
}

void CoreManager::initializeThread(core_id_t core_id)
{
   ScopedLock scoped_maps_lock(m_maps_lock);
   UInt32 tid = getCurrentTID();
   pair<bool, UInt64> e = tid_to_core_map.find(tid);

   LOG_ASSERT_ERROR(e.first == false, "Tried to initialize core %d twice.", core_id);

   const Config::CoreList &core_list = Config::getSingleton()->getCoreListForProcess(Config::getSingleton()->getCurrentProcessNum());

   for (unsigned int i = 0; i < Config::getSingleton()->getNumLocalCores(); i++)
   {
      core_id_t local_core_id = core_list[i];
      if(local_core_id == core_id)
      {
         if (tid_map[i] == UINT_MAX)
         {
            tid_map[i] = tid;
            tid_to_core_index_map.insert(tid, i);
            tid_to_core_map.insert(tid, core_id);
            LOG_PRINT("Initialize thread : index %d mapped to: thread %d, core_id: %d", i, tid_map[i], core_id);
            return;
         }
         else
         {
            LOG_PRINT_ERROR("initializeThread -- %d/%d already mapped to thread %d", i, Config::getSingleton()->getNumLocalCores(), tid_map[i]);
         }
      }
   }

   LOG_PRINT_ERROR("initializeThread - Requested core %d does not live on process %d.", core_id, Config::getSingleton()->getCurrentProcessNum());
}

void CoreManager::terminateThread()
{
   ScopedLock scoped_maps_lock(m_maps_lock);
   UInt32 tid = getCurrentTID();
   LOG_PRINT("CoreManager::terminating thread: %d", tid);
   pair<bool, UInt64> e = tid_to_core_map.find(tid);

   LOG_ASSERT_WARNING(e.first == true, "Thread: %lld not initialized while terminating.", e.second);

   // If it's not in the tid_to_core_map, well then we don't need to remove it
   if(e.first == false)
       return;

   for (UInt32 i = 0; i < Config::getSingleton()->getNumLocalCores(); i++)
   {
      if (tid_map[i] == tid)
      {
         tid_map[i] = UINT_MAX;
         tid_to_core_index_map.remove(tid);
         tid_to_core_map.remove(tid);
         LOG_PRINT("Terminate thread : removed %lld", e.second);
         return;
      }
      else
      {
         // LOG_PRINT("%d/%d already mapped to: %d", i, Config::getSingleton()->getNumLocalCores(), tid_map[i]);
      }
   }

   LOG_PRINT_ERROR("terminateThread - Thread tid: %lld not found in list.", e.second);
}

core_id_t CoreManager::getCurrentCoreID()
{
   core_id_t core_id;
   UInt32 tid = getCurrentTID();

   pair<bool, UInt64> e = tid_to_core_map.find(tid);
   core_id = (e.first == false) ? INVALID_CORE_ID : e.second;

   LOG_ASSERT_ERROR(!e.first || core_id < (SInt32)Config::getSingleton()->getTotalCores(), "Illegal core_id value returned by getCurrentCoreID!\n");

   return core_id;
}

core_id_t CoreManager::getCurrentSimThreadCoreID()
{
   core_id_t core_id;
   UInt32 tid = getCurrentTID();

   pair<bool, UInt64> e = simthread_tid_to_core_map.find(tid);
   core_id = (e.first == false) ? INVALID_CORE_ID : e.second;

   LOG_ASSERT_ERROR(!e.first || core_id < (SInt32)Config::getSingleton()->getTotalCores(), "Illegal core_id value returned by getCurrentCoreID!\n");

   return core_id;
}

Core *CoreManager::getCurrentCore()
{
   Core *core;
   UInt32 tid = getCurrentTID();

   pair<bool, UInt64> e = tid_to_core_index_map.find(tid);
   core = (e.first == false) ? NULL : m_cores[e.second];

   LOG_ASSERT_ERROR(!e.first || e.second < Config::getSingleton()->getTotalCores(), "Illegal core_id value returned by getCurrentCore!\n");
   return core;
}

Core *CoreManager::getCoreFromID(core_id_t id)
{
   Core *core = NULL;
   // Look up the index from the core list
   // FIXME: make this more cached
   const Config::CoreList & cores(Config::getSingleton()->getCoreListForProcess(Config::getSingleton()->getCurrentProcessNum()));
   UInt32 idx = 0;
   for (Config::CLCI i = cores.begin(); i != cores.end(); i++)
   {
      if (*i == id)
      {
         core = m_cores[idx];
         break;
      }

      idx++;
   }

   LOG_ASSERT_ERROR(!core || idx < Config::getSingleton()->getNumLocalCores(), "Illegal index in getCoreFromID!\n");

   return core;
}

Core *CoreManager::getCoreFromIndex(UInt32 index)
{
   LOG_ASSERT_ERROR(index < Config::getSingleton()->getNumLocalCores(), "getCoreFromIndex -- invalid index %d", index);

   return m_cores[index];
}

void CoreManager::outputSummary()
{
   LOG_PRINT("Starting CoreManager::outputSummary");

   ofstream out(Config::getSingleton()->getOutputFileName());

   for (UInt32 i = 0; i < Config::getSingleton()->getNumLocalCores(); i++)
   {
      LOG_PRINT("Output summary core %i", i);

      out << "*** Core[" << i << "] summary ***" << endl;
      if (Config::getSingleton()->getEnablePerformanceModeling())
      {
         m_cores[i]->getPerfModel()->outputSummary(out);
         m_cores[i]->getNetwork()->outputSummary(out);
      }

      if (Config::getSingleton()->getEnableDCacheModeling() || Config::getSingleton()->getEnableICacheModeling())
         m_cores[i]->getOCache()->outputSummary(out);

      out << endl;
   }

   out.close();
}

core_id_t CoreManager::registerSimMemThread()
{
   UInt32 tid = getCurrentTID();

   ScopedLock sl(m_maps_lock);
   pair<bool, UInt64> e = simthread_tid_to_core_map.find(tid);

   // If this thread isn't registered
   if (e.first == false)
   {
      // Search for an unused core to map this simthread thread to
      // one less to account for the MCP
      for (UInt32 i = 0; i < Config::getSingleton()->getNumLocalCores(); i++)
      {
         // Unused slots are set to UINT_MAX
         // FIXME: Use a different constant than UINT_MAX
         if (core_to_simthread_tid_map[i] == UINT_MAX)
         {
            core_to_simthread_tid_map[i] = tid;
            simthread_tid_to_core_map.insert(tid, i);
            return Config::getSingleton()->getCoreListForProcess(Config::getSingleton()->getCurrentProcessNum())[i];
         }
      }

      LOG_PRINT("registerSimMemThread - No free cores for thread: %d", tid);
      for (UInt32 j = 0; j < Config::getSingleton()->getNumLocalCores(); j++)
         LOG_PRINT("core_to_simthread_tid_map[%d] = %d\n", j, core_to_simthread_tid_map[j]);
      LOG_PRINT_ERROR("");
   }
   else
   {
      LOG_PRINT_WARNING("registerSimMemThread - Initialized thread twice");
      // FIXME: I think this is OK
      return simthread_tid_to_core_map.find(tid).second;
   }

   return INVALID_CORE_ID;
}

UInt32 CoreManager::getCurrentTID()
{
   return  syscall(__NR_gettid);
}

<commit_msg>[core_manager] Add bounds checks, remove dead code.<commit_after>#include <sched.h>
#include <linux/unistd.h>
#include <sys/types.h>
#include <sys/syscall.h>
#include <unistd.h>

#include "core_manager.h"
#include "core.h"
#include "network.h"
#include "ocache.h"
#include "config.h"
#include "packetize.h"
#include "message_types.h"

#include "log.h"

using namespace std;

CoreManager::CoreManager()
      :
      tid_to_core_map(3*Config::getSingleton()->getNumLocalCores()),
      tid_to_core_index_map(3*Config::getSingleton()->getNumLocalCores()),
      simthread_tid_to_core_map(3*Config::getSingleton()->getNumLocalCores())
{
   LOG_PRINT("Starting CoreManager Constructor.");

   UInt32 num_local_cores = Config::getSingleton()->getNumLocalCores();

   tid_map = new UInt32 [num_local_cores];
   core_to_simthread_tid_map = new UInt32 [num_local_cores];

   UInt32 proc_id = Config::getSingleton()->getCurrentProcessNum();
   const Config::CoreList &local_cores = Config::getSingleton()->getCoreListForProcess(proc_id);

   for (UInt32 i = 0; i < num_local_cores; i++)
   {
      tid_map[i] = UINT_MAX;
      core_to_simthread_tid_map[i] = UINT_MAX;
      m_cores.push_back(new Core(local_cores[i]));
   }

   LOG_PRINT("Finished CoreManager Constructor.");
}

CoreManager::~CoreManager()
{
   for (std::vector<Core *>::iterator i = m_cores.begin(); i != m_cores.end(); i++)
      delete *i;

   delete [] core_to_simthread_tid_map;
   delete [] tid_map;
}

void CoreManager::initializeCommId(SInt32 comm_id)
{
   UInt32 tid = getCurrentTID();
   pair<bool, UInt64> e = tid_to_core_map.find(tid);

   LOG_ASSERT_ERROR(e.first, "initializeCommId: Called without binding thread to a core.");
   core_id_t core_id = e.second;

   UnstructuredBuffer send_buff;
   send_buff << (SInt32)LCP_MESSAGE_COMMID_UPDATE << comm_id << core_id;

   LOG_PRINT("Initializing comm_id: %d to core_id: %d", comm_id, core_id);

   // Broadcast this update to other processes

   e = tid_to_core_index_map.find(tid);
   LOG_ASSERT_ERROR(e.first, "initializeCommId: tid mapped to core, but not to an index?");
   UInt32 idx = (UInt32) e.second;

   LOG_ASSERT_ERROR(idx < Config::getSingleton()->getNumLocalCores(), "CoreManager got and index [%d] out of range (0-%d).",
         idx, Config::getSingleton()->getNumLocalCores());

   Network *network = m_cores[idx]->getNetwork();
   Transport::Node *transport = Transport::getSingleton()->getGlobalNode();
   UInt32 num_procs = Config::getSingleton()->getProcessCount();

   for (UInt32 i = 0; i < num_procs; i++)
   {
      transport->globalSend(i,
                            send_buff.getBuffer(),
                            send_buff.size());
   }

   LOG_PRINT("Waiting for replies from LCPs.");

   for (UInt32 i = 0; i < num_procs; i++)
   {
      network->netRecvType(LCP_COMM_ID_UPDATE_REPLY);
      LOG_PRINT("Received reply from proc: %d", i);
   }

   LOG_PRINT("Finished.");
}

void CoreManager::initializeThread()
{
   ScopedLock scoped_maps_lock(m_maps_lock);
   UInt32 tid = getCurrentTID();
   pair<bool, UInt64> e = tid_to_core_map.find(tid);

   LOG_ASSERT_WARNING(e.first == false, "Thread: %d already mapped to core: %lld", tid, e.second);

   const Config::CoreList &core_list = Config::getSingleton()->getCoreListForProcess(Config::getSingleton()->getCurrentProcessNum());

   LOG_ASSERT_ERROR(core_list.size() == Config::getSingleton()->getNumLocalCores(),
                    "Core list size different from num local cores? %d != %d", core_list.size(), Config::getSingleton()->getNumLocalCores());

   for (UInt32 i = 0; i < core_list.size(); i++)
   {
      if (tid_map[i] == UINT_MAX)
      {
         core_id_t core_id = core_list[i];
         tid_map[i] = tid;
         tid_to_core_index_map.insert(tid, i);
         tid_to_core_map.insert(tid, core_id);
         LOG_PRINT("Initialize thread : index %d mapped to: thread %d, core_id: %d", i, tid, core_id);
         return;
      }
   }

   LOG_PRINT_ERROR("initializeThread - No free cores out of %d total.", Config::getSingleton()->getNumLocalCores());
}

void CoreManager::initializeThread(core_id_t core_id)
{
   ScopedLock scoped_maps_lock(m_maps_lock);
   UInt32 tid = getCurrentTID();
   pair<bool, UInt64> e = tid_to_core_map.find(tid);

   LOG_ASSERT_ERROR(e.first == false, "Tried to initialize core %d twice.", core_id);

   const Config::CoreList &core_list = Config::getSingleton()->getCoreListForProcess(Config::getSingleton()->getCurrentProcessNum());

   LOG_ASSERT_ERROR(core_list.size() == Config::getSingleton()->getNumLocalCores(),
                    "Core list size different from num local cores? %d != %d", core_list.size(), Config::getSingleton()->getNumLocalCores());

   for (UInt32 i = 0; i < core_list.size(); i++)
   {
      core_id_t local_core_id = core_list[i];
      if(local_core_id == core_id)
      {
         if (tid_map[i] == UINT_MAX)
         {
            tid_map[i] = tid;
            tid_to_core_index_map.insert(tid, i);
            tid_to_core_map.insert(tid, core_id);
            LOG_PRINT("Initialize thread : index %d mapped to: thread %d, core_id: %d", i, tid_map[i], core_id);
            return;
         }
         else
         {
            LOG_PRINT_ERROR("initializeThread -- %d/%d already mapped to thread %d", i, Config::getSingleton()->getNumLocalCores(), tid_map[i]);
         }
      }
   }

   LOG_PRINT_ERROR("initializeThread - Requested core %d does not live on process %d.", core_id, Config::getSingleton()->getCurrentProcessNum());
}

void CoreManager::terminateThread()
{
   ScopedLock scoped_maps_lock(m_maps_lock);
   UInt32 tid = getCurrentTID();
   LOG_PRINT("CoreManager::terminating thread: %d", tid);
   pair<bool, UInt64> e = tid_to_core_map.find(tid);

   LOG_ASSERT_WARNING(e.first == true, "Thread: %lld not initialized while terminating.", e.second);

   // If it's not in the tid_to_core_map, well then we don't need to remove it
   if(e.first == false)
       return;

   for (UInt32 i = 0; i < Config::getSingleton()->getNumLocalCores(); i++)
   {
      if (tid_map[i] == tid)
      {
         tid_map[i] = UINT_MAX;
         tid_to_core_index_map.remove(tid);
         tid_to_core_map.remove(tid);
         LOG_PRINT("Terminate thread : removed %lld", e.second);
         return;
      }
   }

   LOG_PRINT_ERROR("terminateThread - Thread tid: %lld not found in list.", e.second);
}

core_id_t CoreManager::getCurrentCoreID()
{
   core_id_t core_id;
   UInt32 tid = getCurrentTID();

   pair<bool, UInt64> e = tid_to_core_map.find(tid);
   core_id = (e.first == false) ? INVALID_CORE_ID : e.second;

   LOG_ASSERT_ERROR(!e.first || core_id < (SInt32)Config::getSingleton()->getTotalCores(), "Illegal core_id value returned by getCurrentCoreID!\n");

   return core_id;
}

core_id_t CoreManager::getCurrentSimThreadCoreID()
{
   core_id_t core_id;
   UInt32 tid = getCurrentTID();

   pair<bool, UInt64> e = simthread_tid_to_core_map.find(tid);
   core_id = (e.first == false) ? INVALID_CORE_ID : e.second;

   LOG_ASSERT_ERROR(!e.first || core_id < (SInt32)Config::getSingleton()->getTotalCores(), "Illegal core_id value returned by getCurrentCoreID!\n");

   return core_id;
}

Core *CoreManager::getCurrentCore()
{
   Core *core;
   UInt32 tid = getCurrentTID();

   pair<bool, UInt64> e = tid_to_core_index_map.find(tid);
   core = (e.first == false) ? NULL : m_cores[e.second];

   LOG_ASSERT_ERROR(!e.first || e.second < Config::getSingleton()->getTotalCores(), "Illegal core_id value returned by getCurrentCore!\n");
   return core;
}

Core *CoreManager::getCoreFromID(core_id_t id)
{
   Core *core = NULL;
   // Look up the index from the core list
   // FIXME: make this more cached
   const Config::CoreList & cores(Config::getSingleton()->getCoreListForProcess(Config::getSingleton()->getCurrentProcessNum()));
   UInt32 idx = 0;
   for (Config::CLCI i = cores.begin(); i != cores.end(); i++)
   {
      if (*i == id)
      {
         core = m_cores[idx];
         break;
      }

      idx++;
   }

   LOG_ASSERT_ERROR(!core || idx < Config::getSingleton()->getNumLocalCores(), "Illegal index in getCoreFromID!\n");

   return core;
}

Core *CoreManager::getCoreFromIndex(UInt32 index)
{
   LOG_ASSERT_ERROR(index < Config::getSingleton()->getNumLocalCores(), "getCoreFromIndex -- invalid index %d", index);

   return m_cores[index];
}

void CoreManager::outputSummary()
{
   LOG_PRINT("Starting CoreManager::outputSummary");

   ofstream out(Config::getSingleton()->getOutputFileName());

   for (UInt32 i = 0; i < Config::getSingleton()->getNumLocalCores(); i++)
   {
      LOG_PRINT("Output summary core %i", i);

      out << "*** Core[" << i << "] summary ***" << endl;
      if (Config::getSingleton()->getEnablePerformanceModeling())
      {
         m_cores[i]->getPerfModel()->outputSummary(out);
         m_cores[i]->getNetwork()->outputSummary(out);
      }

      if (Config::getSingleton()->getEnableDCacheModeling() || Config::getSingleton()->getEnableICacheModeling())
         m_cores[i]->getOCache()->outputSummary(out);

      out << endl;
   }

   out.close();
}

core_id_t CoreManager::registerSimMemThread()
{
   UInt32 tid = getCurrentTID();

   ScopedLock sl(m_maps_lock);
   pair<bool, UInt64> e = simthread_tid_to_core_map.find(tid);

   // If this thread isn't registered
   if (e.first == false)
   {
      // Search for an unused core to map this simthread thread to
      // one less to account for the MCP
      for (UInt32 i = 0; i < Config::getSingleton()->getNumLocalCores(); i++)
      {
         // Unused slots are set to UINT_MAX
         // FIXME: Use a different constant than UINT_MAX
         if (core_to_simthread_tid_map[i] == UINT_MAX)
         {
            core_to_simthread_tid_map[i] = tid;
            simthread_tid_to_core_map.insert(tid, i);
            return Config::getSingleton()->getCoreListForProcess(Config::getSingleton()->getCurrentProcessNum())[i];
         }
      }

      LOG_PRINT("registerSimMemThread - No free cores for thread: %d", tid);
      for (UInt32 j = 0; j < Config::getSingleton()->getNumLocalCores(); j++)
         LOG_PRINT("core_to_simthread_tid_map[%d] = %d\n", j, core_to_simthread_tid_map[j]);
      LOG_PRINT_ERROR("");
   }
   else
   {
      LOG_PRINT_WARNING("registerSimMemThread - Initialized thread twice");
      // FIXME: I think this is OK
      return simthread_tid_to_core_map.find(tid).second;
   }

   return INVALID_CORE_ID;
}

UInt32 CoreManager::getCurrentTID()
{
   return  syscall(__NR_gettid);
}

<|endoftext|>


<commit_before>#include "cnn/init.h"
#include "cnn/aligned-mem-pool.h"
#include "cnn/cnn.h"
#include "cnn/model.h"
#include <iostream>
#include <random>
#include <cmath>

#if HAVE_CUDA
#include "cnn/cuda.h"
#include <device_launch_parameters.h>
#endif

using namespace std;

namespace cnn {

    AlignedMemoryPool<ALIGN>* fxs = nullptr;
    AlignedMemoryPool<ALIGN>* dEdfs = nullptr;
    AlignedMemoryPool<ALIGN>* mem_nodes= nullptr;   /// for nodes allocation/delocation. operation of new/delete of each node has been overwritten to use this memory pool for speed-up
    AlignedMemoryPool<ALIGN>* glb_temp_working_mem = nullptr;
    AlignedMemoryPool<ALIGN>* glb_temp_lookup_gradient_value_mem = nullptr; /// this saves gradient on those sparse lookup table parameters that have non-zero gradiens. these values and gradients are temporary
    mt19937* rndeng = nullptr;

    char* getCmdOption(char ** begin, char ** end, const std::string & option)
    {
        char ** itr = std::find(begin, end, option);
        if (itr != end && ++itr != end)
        {
            return *itr;
        }
        return 0;
    }

	static void RemoveArgs(int& argc, char**& argv, int& argi, int n) {
	  for (int i = argi + n; i < argc; ++i)
	    argv[i - n] = argv[i];
	  argc -= n;
	  assert(argc >= 0);
	}

    bool cmdOptionExists(char** begin, char** end, const std::string& option)
    {
        return std::find(begin, end, option) != end;
    }

    void Initialize(int& argc, char**& argv, unsigned random_seed, bool demo) {
        cerr << "Initializing...\n";
#if HAVE_CUDA
        Initialize_GPU(argc, argv);
#else
        kSCALAR_MINUSONE = (cnn::real*)cnn_mm_malloc(sizeof(cnn::real), CNN_ALIGN);
        *kSCALAR_MINUSONE = -1;
        kSCALAR_ONE = (cnn::real*)cnn_mm_malloc(sizeof(cnn::real), CNN_ALIGN);
        *kSCALAR_ONE = 1;
        kSCALAR_ZERO = (cnn::real*)cnn_mm_malloc(sizeof(cnn::real), CNN_ALIGN);
        *kSCALAR_ZERO = 0;
#endif

        if (random_seed == 0)
        {
            if (cmdOptionExists(argv, argv + argc, "--seed"))
            {
                string seed = getCmdOption(argv, argv + argc, "--seed");
                stringstream(seed) >> random_seed;
            }
            else
            {
                random_device rd;
                random_seed = rd();
            }
        }
        rndeng = new mt19937(random_seed);

        cerr << "Allocating memory...\n";
		unsigned long num_mb = 512UL;
        mem_nodes = new AlignedMemoryPool<ALIGN>(512UL * (1UL << 20), true);
        glb_temp_working_mem = new AlignedMemoryPool<ALIGN>(1UL << 16);
        glb_temp_lookup_gradient_value_mem = new AlignedMemoryPool<ALIGN>(1UL << 26);

        if (demo)
        {
            fxs = new AlignedMemoryPool<ALIGN>(512UL * (1UL << 20));
            dEdfs = new AlignedMemoryPool<ALIGN>(512UL * (1UL << 20));
        }
        else
        {
#ifdef HAVE_CUDA
            fxs = new AlignedMemoryPool<ALIGN>(512UL * (1UL << 22));
            dEdfs = new AlignedMemoryPool<ALIGN>(512UL * (1UL << 22));
#else
            fxs = new AlignedMemoryPool<ALIGN>(512UL * (1UL << 22));
            dEdfs = new AlignedMemoryPool<ALIGN>(512UL * (1UL << 22));
#endif
        }
        cerr << "Done.\n";
    }

  void Free()
  {
        cerr << "Freeing memory ...\n";
        cnn_mm_free(kSCALAR_MINUSONE);
        cnn_mm_free(kSCALAR_ONE);
        cnn_mm_free(kSCALAR_ZERO);

        delete (rndeng);
        delete (fxs);
        delete (dEdfs);
        delete (mem_nodes);
        delete (glb_temp_working_mem);

        for (auto p : kSCALAR_ONE_OVER_INT)
            cnn_mm_free(p);

#ifdef HAVE_CUDA
        Free_GPU();
#endif
        cerr << "Done.\n";
  }

} // namespace cnn
<commit_msg>smaller space for temp gradient and value space as value is not need to be allocated for non-zero gradients<commit_after>#include "cnn/init.h"
#include "cnn/aligned-mem-pool.h"
#include "cnn/cnn.h"
#include "cnn/model.h"
#include <iostream>
#include <random>
#include <cmath>

#if HAVE_CUDA
#include "cnn/cuda.h"
#include <device_launch_parameters.h>
#endif

using namespace std;

namespace cnn {

    AlignedMemoryPool<ALIGN>* fxs = nullptr;
    AlignedMemoryPool<ALIGN>* dEdfs = nullptr;
    AlignedMemoryPool<ALIGN>* mem_nodes= nullptr;   /// for nodes allocation/delocation. operation of new/delete of each node has been overwritten to use this memory pool for speed-up
    AlignedMemoryPool<ALIGN>* glb_temp_working_mem = nullptr;
    AlignedMemoryPool<ALIGN>* glb_temp_lookup_gradient_value_mem = nullptr; /// this saves gradient on those sparse lookup table parameters that have non-zero gradiens. these values and gradients are temporary
    mt19937* rndeng = nullptr;

    char* getCmdOption(char ** begin, char ** end, const std::string & option)
    {
        char ** itr = std::find(begin, end, option);
        if (itr != end && ++itr != end)
        {
            return *itr;
        }
        return 0;
    }

	static void RemoveArgs(int& argc, char**& argv, int& argi, int n) {
	  for (int i = argi + n; i < argc; ++i)
	    argv[i - n] = argv[i];
	  argc -= n;
	  assert(argc >= 0);
	}

    bool cmdOptionExists(char** begin, char** end, const std::string& option)
    {
        return std::find(begin, end, option) != end;
    }

    void Initialize(int& argc, char**& argv, unsigned random_seed, bool demo) {
        cerr << "Initializing...\n";
#if HAVE_CUDA
        Initialize_GPU(argc, argv);
#else
        kSCALAR_MINUSONE = (cnn::real*)cnn_mm_malloc(sizeof(cnn::real), CNN_ALIGN);
        *kSCALAR_MINUSONE = -1;
        kSCALAR_ONE = (cnn::real*)cnn_mm_malloc(sizeof(cnn::real), CNN_ALIGN);
        *kSCALAR_ONE = 1;
        kSCALAR_ZERO = (cnn::real*)cnn_mm_malloc(sizeof(cnn::real), CNN_ALIGN);
        *kSCALAR_ZERO = 0;
#endif

        if (random_seed == 0)
        {
            if (cmdOptionExists(argv, argv + argc, "--seed"))
            {
                string seed = getCmdOption(argv, argv + argc, "--seed");
                stringstream(seed) >> random_seed;
            }
            else
            {
                random_device rd;
                random_seed = rd();
            }
        }
        rndeng = new mt19937(random_seed);

        cerr << "Allocating memory...\n";
		unsigned long num_mb = 512UL;
        mem_nodes = new AlignedMemoryPool<ALIGN>(512UL * (1UL << 20), true);
        glb_temp_working_mem = new AlignedMemoryPool<ALIGN>(1UL << 16);
        glb_temp_lookup_gradient_value_mem = new AlignedMemoryPool<ALIGN>(1UL << 25);

        if (demo)
        {
            fxs = new AlignedMemoryPool<ALIGN>(512UL * (1UL << 20));
            dEdfs = new AlignedMemoryPool<ALIGN>(512UL * (1UL << 20));
        }
        else
        {
#ifdef HAVE_CUDA
            fxs = new AlignedMemoryPool<ALIGN>(512UL * (1UL << 22));
            dEdfs = new AlignedMemoryPool<ALIGN>(512UL * (1UL << 22));
#else
            fxs = new AlignedMemoryPool<ALIGN>(512UL * (1UL << 22));
            dEdfs = new AlignedMemoryPool<ALIGN>(512UL * (1UL << 22));
#endif
        }
        cerr << "Done.\n";
    }

  void Free()
  {
        cerr << "Freeing memory ...\n";
        cnn_mm_free(kSCALAR_MINUSONE);
        cnn_mm_free(kSCALAR_ONE);
        cnn_mm_free(kSCALAR_ZERO);

        delete (rndeng);
        delete (fxs);
        delete (dEdfs);
        delete (mem_nodes);
        delete (glb_temp_working_mem);

        for (auto p : kSCALAR_ONE_OVER_INT)
            cnn_mm_free(p);

#ifdef HAVE_CUDA
        Free_GPU();
#endif
        cerr << "Done.\n";
  }

} // namespace cnn
<|endoftext|>