<commit_before>#pragma once

#include <depthai-shared/common/EepromData.hpp>
#include <depthai-shared/common/optional.hpp>
#include <nlohmann/json.hpp>

#include "depthai-shared/common/CameraBoardSocket.hpp"
#include "depthai-shared/datatype/RawStereoDepthConfig.hpp"

namespace dai {

/**
 * Specify properties for StereoDepth
 */
struct StereoDepthProperties {
    struct RectificationMesh {
        /**
         * Uri which points to the mesh array for 'left' input rectification
         */
        std::string meshLeftUri;
        /**
         * Uri which points to the mesh array for 'right' input rectification
         */
        std::string meshRightUri;
        /**
         * Mesh array size in bytes, for each of 'left' and 'right' (need to match)
         */
        tl::optional<std::uint32_t> meshSize;
        /**
         * Distance between mesh points, in the horizontal direction
         */
        uint16_t stepWidth = 16;
        /**
         * Distance between mesh points, in the vertical direction
         */
        uint16_t stepHeight = 16;

        NLOHMANN_DEFINE_TYPE_INTRUSIVE(RectificationMesh, meshLeftUri, meshRightUri, meshSize, stepWidth, stepHeight);
    };

    /// Initial stereo config
    RawStereoDepthConfig initialConfig;

    /// Whether to wait for config at 'inputConfig' IO
    bool inputConfigSync = false;

    using MedianFilter = dai::MedianFilter;

    /**
     * Align the disparity/depth to the perspective of a rectified output, or center it
     */
    enum class DepthAlign : int32_t { RECTIFIED_RIGHT, RECTIFIED_LEFT, CENTER };

    /**
     * Calibration data byte array
     */
    std::vector<std::uint8_t> calibration;

    EepromData calibrationData;

    /**
     * Set the disparity/depth alignment to the perspective of a rectified output, or center it
     */
    DepthAlign depthAlign = DepthAlign::RECTIFIED_RIGHT;
    /**
     * Which camera to align disparity/depth to.
     * When configured (not AUTO), takes precedence over 'depthAlign'
     */
    CameraBoardSocket depthAlignCamera = CameraBoardSocket::AUTO;

    bool enableRectification = true;
    /**
     * Computes and combines disparities in both L-R and R-L directions, and combine them.
     * For better occlusion handling
     */
    bool enableLeftRightCheck = false;
    /**
     * Computes disparity with sub-pixel interpolation (5 fractional bits), suitable for long range
     */
    bool enableSubpixel = false;
    /**
     * Disparity range increased from 96 to 192, combined from full resolution and downscaled images.
     * Suitable for short range objects
     */
    bool enableExtendedDisparity = false;
    /**
     * Mirror rectified frames: true to have disparity/depth normal (non-mirrored)
     */
    bool rectifyMirrorFrame = true;
    /**
     * Fill color for missing data at frame edges: grayscale 0..255, or -1 to replicate pixels
     */
    std::int32_t rectifyEdgeFillColor = -1;
    /**
     * Input frame width. Optional (taken from MonoCamera nodes if they exist)
     */
    tl::optional<std::int32_t> width;
    /**
     * Input frame height. Optional (taken from MonoCamera nodes if they exist)
     */
    tl::optional<std::int32_t> height;
    /**
     * Output disparity/depth width. Currently only used when aligning to RGB
     */
    tl::optional<std::int32_t> outWidth;
    /**
     * Output disparity/depth height. Currently only used when aligning to RGB
     */
    tl::optional<std::int32_t> outHeight;
    /**
     * Whether to keep aspect ratio of the input (rectified) or not
     */
    bool outKeepAspectRatio = true;

    /**
     * Specify a direct warp mesh to be used for rectification,
     * instead of intrinsics + extrinsic matrices
     */
    RectificationMesh mesh;
};

NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(StereoDepthProperties,
                                   initialConfig,
                                   inputConfigSync,
                                   calibration,
                                   calibrationData,
                                   depthAlign,
                                   depthAlignCamera,
                                   enableRectification,
                                   enableLeftRightCheck,
                                   enableSubpixel,
                                   enableExtendedDisparity,
                                   rectifyMirrorFrame,
                                   rectifyEdgeFillColor,
                                   width,
                                   height,
                                   outWidth,
                                   outHeight,
                                   outKeepAspectRatio,
                                   mesh);

}  // namespace dai
<commit_msg>Make stereo host calibration optional<commit_after>#pragma once

#include <depthai-shared/common/EepromData.hpp>
#include <depthai-shared/common/optional.hpp>
#include <nlohmann/json.hpp>

#include "depthai-shared/common/CameraBoardSocket.hpp"
#include "depthai-shared/datatype/RawStereoDepthConfig.hpp"

namespace dai {

/**
 * Specify properties for StereoDepth
 */
struct StereoDepthProperties {
    struct RectificationMesh {
        /**
         * Uri which points to the mesh array for 'left' input rectification
         */
        std::string meshLeftUri;
        /**
         * Uri which points to the mesh array for 'right' input rectification
         */
        std::string meshRightUri;
        /**
         * Mesh array size in bytes, for each of 'left' and 'right' (need to match)
         */
        tl::optional<std::uint32_t> meshSize;
        /**
         * Distance between mesh points, in the horizontal direction
         */
        uint16_t stepWidth = 16;
        /**
         * Distance between mesh points, in the vertical direction
         */
        uint16_t stepHeight = 16;

        NLOHMANN_DEFINE_TYPE_INTRUSIVE(RectificationMesh, meshLeftUri, meshRightUri, meshSize, stepWidth, stepHeight);
    };

    /// Initial stereo config
    RawStereoDepthConfig initialConfig;

    /// Whether to wait for config at 'inputConfig' IO
    bool inputConfigSync = false;

    using MedianFilter = dai::MedianFilter;

    /**
     * Align the disparity/depth to the perspective of a rectified output, or center it
     */
    enum class DepthAlign : int32_t { RECTIFIED_RIGHT, RECTIFIED_LEFT, CENTER };

    /**
     * Calibration data byte array
     */
    tl::optional<std::vector<std::uint8_t>> calibration;

    EepromData calibrationData;

    /**
     * Set the disparity/depth alignment to the perspective of a rectified output, or center it
     */
    DepthAlign depthAlign = DepthAlign::RECTIFIED_RIGHT;
    /**
     * Which camera to align disparity/depth to.
     * When configured (not AUTO), takes precedence over 'depthAlign'
     */
    CameraBoardSocket depthAlignCamera = CameraBoardSocket::AUTO;

    bool enableRectification = true;
    /**
     * Computes and combines disparities in both L-R and R-L directions, and combine them.
     * For better occlusion handling
     */
    bool enableLeftRightCheck = false;
    /**
     * Computes disparity with sub-pixel interpolation (5 fractional bits), suitable for long range
     */
    bool enableSubpixel = false;
    /**
     * Disparity range increased from 96 to 192, combined from full resolution and downscaled images.
     * Suitable for short range objects
     */
    bool enableExtendedDisparity = false;
    /**
     * Mirror rectified frames: true to have disparity/depth normal (non-mirrored)
     */
    bool rectifyMirrorFrame = true;
    /**
     * Fill color for missing data at frame edges: grayscale 0..255, or -1 to replicate pixels
     */
    std::int32_t rectifyEdgeFillColor = -1;
    /**
     * Input frame width. Optional (taken from MonoCamera nodes if they exist)
     */
    tl::optional<std::int32_t> width;
    /**
     * Input frame height. Optional (taken from MonoCamera nodes if they exist)
     */
    tl::optional<std::int32_t> height;
    /**
     * Output disparity/depth width. Currently only used when aligning to RGB
     */
    tl::optional<std::int32_t> outWidth;
    /**
     * Output disparity/depth height. Currently only used when aligning to RGB
     */
    tl::optional<std::int32_t> outHeight;
    /**
     * Whether to keep aspect ratio of the input (rectified) or not
     */
    bool outKeepAspectRatio = true;

    /**
     * Specify a direct warp mesh to be used for rectification,
     * instead of intrinsics + extrinsic matrices
     */
    RectificationMesh mesh;
};

NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(StereoDepthProperties,
                                   initialConfig,
                                   inputConfigSync,
                                   calibration,
                                   calibrationData,
                                   depthAlign,
                                   depthAlignCamera,
                                   enableRectification,
                                   enableLeftRightCheck,
                                   enableSubpixel,
                                   enableExtendedDisparity,
                                   rectifyMirrorFrame,
                                   rectifyEdgeFillColor,
                                   width,
                                   height,
                                   outWidth,
                                   outHeight,
                                   outKeepAspectRatio,
                                   mesh);

}  // namespace dai
<|endoftext|>


<commit_before> /*
 * Copyright deipi.com LLC and contributors. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "utils.h"

#include "database.h"
#include <xapian/dbfactory.h>

#define DOCUMENT_ID_TERM_PREFIX 'Q'

Database::Database(Endpoints &endpoints_, bool writable_)
	: endpoints(endpoints_),
	  writable(writable_),
	  db(NULL)
{
	hash = endpoints.hash(writable);
	reopen();
}

void
Database::reopen()
{
	if (db) {
		// Try to reopen
		try {
			db->reopen();
			return;
		} catch (const Xapian::Error &e) {
			LOG_ERR(this, "ERROR: %s\n", e.get_msg().c_str());
			db->close();
			delete db;
			db = NULL;
		}
	}

	// FIXME: Handle remote endpoints and figure out if the endpoint is a local database
	const Endpoint *e;
	if (writable) {
		db = new Xapian::WritableDatabase();
		if (endpoints.size() != 1) {
			LOG_ERR(this, "ERROR: Expecting exactly one database, %d requested: %s", endpoints.size(), endpoints.as_string().c_str());
		} else {
			e = &endpoints[0];
			if (e->protocol == "file") {
				db->add_database(Xapian::WritableDatabase(e->path, Xapian::DB_CREATE_OR_OPEN));
			} else {
				db->add_database(Xapian::Remote::open_writable(e->host, e->port, 0, 10000, e->path));
			}
		}
	} else {
		db = new Xapian::Database();
		std::vector<Endpoint>::const_iterator i(endpoints.begin());
		for (; i != endpoints.end(); ++i) {
			e = &(*i);
			if (e->protocol == "file") {
				db->add_database(Xapian::Database(e->path, Xapian::DB_CREATE_OR_OPEN));
			} else {
				db->add_database(Xapian::Remote::open(e->host, e->port, 0, 10000, e->path));
			}
		}
	}
}

Database::~Database()
{
	delete db;
}

DatabaseQueue::DatabaseQueue()
	: count(0)
{
}

DatabaseQueue::~DatabaseQueue()
{
	while (!empty()) {
		Database *database;
		if (pop(database)) {
			delete database;
		}
	}
}

DatabasePool::DatabasePool()
	: finished(false)
{
	pthread_mutexattr_init(&qmtx_attr);
	pthread_mutexattr_settype(&qmtx_attr, PTHREAD_MUTEX_RECURSIVE);
	pthread_mutex_init(&qmtx, &qmtx_attr);
}

DatabasePool::~DatabasePool()
{
	finish();

	pthread_mutex_destroy(&qmtx);
	pthread_mutexattr_destroy(&qmtx_attr);
}

void DatabasePool::finish() {
	pthread_mutex_lock(&qmtx);

	finished = true;

	pthread_mutex_unlock(&qmtx);
}

bool
DatabasePool::checkout(Database **database, Endpoints &endpoints, bool writable)
{
	Database *database_ = NULL;

	LOG_DATABASE(this, "+ CHECKING OUT DB %lx %s(%s)...\n", (unsigned long)*database, writable ? "w" : "r", endpoints.as_string().c_str());

	pthread_mutex_lock(&qmtx);

	if (!finished && *database == NULL) {
		size_t hash = endpoints.hash(writable);
		DatabaseQueue &queue = databases[hash];

		if (!queue.pop(database_, 0)) {
			if (!writable || queue.count == 0) {
				queue.count++;
				pthread_mutex_unlock(&qmtx);
				database_ = new Database(endpoints, writable);
				pthread_mutex_lock(&qmtx);
			} else {
				// Lock until a database is available if it can't get one.
				pthread_mutex_unlock(&qmtx);
				int s = queue.pop(database_);
				pthread_mutex_lock(&qmtx);
				if (!s) {
					LOG_ERR(this, "ERROR: Database is not available. Writable: %d", writable);
				}
			}
		}
		*database = database_;
	}

	pthread_mutex_unlock(&qmtx);

	LOG_DATABASE(this, "+ CHECKOUT DB %lx\n", (unsigned long)*database);

	return database_ != NULL;
}

void
DatabasePool::checkin(Database **database)
{
	LOG_DATABASE(this, "- CHECKING IN DB %lx %s(%s)...\n", (unsigned long)*database, (*database)->writable ? "w" : "r", (*database)->endpoints.as_string().c_str());

	pthread_mutex_lock(&qmtx);

	DatabaseQueue &queue = databases[(*database)->hash];

	queue.push(*database);

	*database = NULL;

	pthread_mutex_unlock(&qmtx);

	LOG_DATABASE(this, "- CHECKIN DB %lx\n", (unsigned long)*database);
}

void
Database::drop(const char *document_id, bool commit)
{
	if (!writable) {
		LOG_ERR(this, "ERROR: database is %s\n", writable ? "w" : "r");
		return;
	}

    document_id  = prefixed(document_id, DOCUMENT_ID_TERM_PREFIX);

	for (int t = 3; t >= 0; --t) {
		LOG(this, "Deleting: -%s- t:%d\n", document_id, t);
		Xapian::WritableDatabase *wdb = static_cast<Xapian::WritableDatabase *>(db);
	    try {
	    	wdb->delete_document(document_id);
	    } catch (const Xapian::Error &e) {
			LOG_ERR(this, "ERROR: %s\n", e.get_msg().c_str());
			if (t) reopen();
			continue;
		}
    	LOG(this, "Document deleted\n");
		if (commit) _commit();
		return;
	}

	LOG_ERR(this, "ERROR: Cannot delete document: %s!\n", document_id);
}

char*
Database::prefixed(const char *term, const char prefix)
{
    char *prefix_term = new char[strlen(term) + 2];
    int i = 0;

    prefix_term[0] = toupper(prefix);
    prefix_term[1] = ':';

    while (term[i] != '\0') {
        prefix_term[i+2] = term[i];
        i++;
    }
    prefix_term[i+2] = '\0';
    return prefix_term;
}

char*
Database::prefixed_string(const char *term, const char *prefix)
{
    char *prefix_term = new char[strlen(term) + strlen(prefix)];
    int i = 0, j = 0;

    while (prefix[i] != '\0') {
  	  	prefix_term[i] = toupper(prefix[i]);
  	  	i++;
    }
    while (term[j] != '\0') {
        prefix_term[i] = term[j];
        i++;
        j++;
    }
    return prefix_term;
}

void
Database::_commit()
{
	for (int t = 3; t >= 0; --t) {
		LOG(this, "Commit: t%d\n", t);
		Xapian::WritableDatabase *wdb = static_cast<Xapian::WritableDatabase *>(db);
		try {
			wdb->commit();
		} catch (const Xapian::Error &e) {
			LOG_ERR(this, "ERROR: %s\n", e.get_msg().c_str());
			if (t) reopen();
			continue;
		}

		LOG(this, "Commit made\n");
		return;
	}

	LOG_ERR(this, "ERROR: Cannot do commit!\n");
}

void
Database::index(const char *document, bool commit)
{
	if (!writable) {
		LOG_ERR(this, "ERROR: database is %s\n", writable ? "w" : "r");
		return;
	}

	Xapian::WritableDatabase *wdb = static_cast<Xapian::WritableDatabase *>(db);

	cJSON *root = cJSON_Parse(document);
	if (!root) {
		printf("Error before: [%s]\n",cJSON_GetErrorPtr());
	}
	cJSON *doc_id = root ? cJSON_GetObjectItem(root, "id") : NULL;
    cJSON *document_data = root ? cJSON_GetObjectItem(root, "data") : NULL;
    cJSON *document_values = root ? cJSON_GetObjectItem(root, "values") : NULL;
    cJSON *document_terms = root ? cJSON_GetObjectItem(root, "terms") : NULL;
  	LOG(this, "Document_terms: %s\n", cJSON_Print(document_terms));

    Xapian::Document doc;

    LOG(this, "Start DATA");
    if (document_data) {
    	const char *doc_data = cJSON_Print(document_data);
    	LOG(this, "Document data: %s\n", doc_data);
		doc.set_data(doc_data);
    }
    LOG(this, "End DATA");

    //Document Values

    LOG(this, "Start values");
    if (document_values) {
    	for (int i = 0; i < cJSON_GetArraySize(document_values); i++) {
    		cJSON *value = cJSON_GetArrayItem(document_values, i);
            if (value->type == 3) {
            	doc.add_value((i+1), Xapian::sortable_serialise(value->valuedouble));
            	LOG(this, "%s: %f\n", value->string, value->valuedouble);
            } else {
            	LOG_ERR(this, "ERROR: %s must be a double (%s)\n", value->string, cJSON_Print(value));
            }
  		}
    }
    LOG(this, "End values");

    //Make sure document_id is also a term (otherwise it doesn't replace an existing document)
    const char *document_id  = prefixed(doc_id->valuestring, DOCUMENT_ID_TERM_PREFIX);
    doc.add_value(0, doc_id->valuestring);
    doc.add_boolean_term(document_id);

    //Document Terms
    LOG(this, "Document terms: %s\n", cJSON_Print(document_terms));
    if (document_terms) {
    	LOG(this, "Terms..\n");
    	for (int i = 0; i < cJSON_GetArraySize(document_terms); i++) {
            cJSON *term = cJSON_GetArrayItem(document_terms, i);
            LOG(this, "Prefix: %s   Term: %s\n",term->string, term->valuestring);
            doc.add_term(prefixed_string(term->valuestring, term->string), 1);
            LOG(this, "Term: %s was added\n", prefixed_string(term->valuestring, term->string));
  		}
    }

    cJSON_Delete(root);
}<commit_msg>Concatenating a string with a character, and string concatenation<commit_after>/*
 * Copyright deipi.com LLC and contributors. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "utils.h"

#include "database.h"
#include <xapian/dbfactory.h>

#define DOCUMENT_ID_TERM_PREFIX 'Q'
#define DOCUMENT_CUSTOM_TERM_PREFIX = 'X'

Database::Database(Endpoints &endpoints_, bool writable_)
	: endpoints(endpoints_),
	  writable(writable_),
	  db(NULL)
{
	hash = endpoints.hash(writable);
	reopen();
}

void
Database::reopen()
{
	if (db) {
		// Try to reopen
		try {
			db->reopen();
			return;
		} catch (const Xapian::Error &e) {
			LOG_ERR(this, "ERROR: %s\n", e.get_msg().c_str());
			db->close();
			delete db;
			db = NULL;
		}
	}

	// FIXME: Handle remote endpoints and figure out if the endpoint is a local database
	const Endpoint *e;
	if (writable) {
		db = new Xapian::WritableDatabase();
		if (endpoints.size() != 1) {
			LOG_ERR(this, "ERROR: Expecting exactly one database, %d requested: %s", endpoints.size(), endpoints.as_string().c_str());
		} else {
			e = &endpoints[0];
			if (e->protocol == "file") {
				db->add_database(Xapian::WritableDatabase(e->path, Xapian::DB_CREATE_OR_OPEN));
			} else {
				db->add_database(Xapian::Remote::open_writable(e->host, e->port, 0, 10000, e->path));
			}
		}
	} else {
		db = new Xapian::Database();
		std::vector<Endpoint>::const_iterator i(endpoints.begin());
		for (; i != endpoints.end(); ++i) {
			e = &(*i);
			if (e->protocol == "file") {
				db->add_database(Xapian::Database(e->path, Xapian::DB_CREATE_OR_OPEN));
			} else {
				db->add_database(Xapian::Remote::open(e->host, e->port, 0, 10000, e->path));
			}
		}
	}
}

Database::~Database()
{
	delete db;
}

DatabaseQueue::DatabaseQueue()
	: count(0)
{
}

DatabaseQueue::~DatabaseQueue()
{
	while (!empty()) {
		Database *database;
		if (pop(database)) {
			delete database;
		}
	}
}

DatabasePool::DatabasePool()
	: finished(false)
{
	pthread_mutexattr_init(&qmtx_attr);
	pthread_mutexattr_settype(&qmtx_attr, PTHREAD_MUTEX_RECURSIVE);
	pthread_mutex_init(&qmtx, &qmtx_attr);
}

DatabasePool::~DatabasePool()
{
	finish();

	pthread_mutex_destroy(&qmtx);
	pthread_mutexattr_destroy(&qmtx_attr);
}

void DatabasePool::finish() {
	pthread_mutex_lock(&qmtx);

	finished = true;

	pthread_mutex_unlock(&qmtx);
}

bool
DatabasePool::checkout(Database **database, Endpoints &endpoints, bool writable)
{
	Database *database_ = NULL;

	LOG_DATABASE(this, "+ CHECKING OUT DB %lx %s(%s)...\n", (unsigned long)*database, writable ? "w" : "r", endpoints.as_string().c_str());

	pthread_mutex_lock(&qmtx);

	if (!finished && *database == NULL) {
		size_t hash = endpoints.hash(writable);
		DatabaseQueue &queue = databases[hash];

		if (!queue.pop(database_, 0)) {
			if (!writable || queue.count == 0) {
				queue.count++;
				pthread_mutex_unlock(&qmtx);
				database_ = new Database(endpoints, writable);
				pthread_mutex_lock(&qmtx);
			} else {
				// Lock until a database is available if it can't get one.
				pthread_mutex_unlock(&qmtx);
				int s = queue.pop(database_);
				pthread_mutex_lock(&qmtx);
				if (!s) {
					LOG_ERR(this, "ERROR: Database is not available. Writable: %d", writable);
				}
			}
		}
		*database = database_;
	}

	pthread_mutex_unlock(&qmtx);

	LOG_DATABASE(this, "+ CHECKOUT DB %lx\n", (unsigned long)*database);

	return database_ != NULL;
}

void
DatabasePool::checkin(Database **database)
{
	LOG_DATABASE(this, "- CHECKING IN DB %lx %s(%s)...\n", (unsigned long)*database, (*database)->writable ? "w" : "r", (*database)->endpoints.as_string().c_str());

	pthread_mutex_lock(&qmtx);

	DatabaseQueue &queue = databases[(*database)->hash];

	queue.push(*database);

	*database = NULL;

	pthread_mutex_unlock(&qmtx);

	LOG_DATABASE(this, "- CHECKIN DB %lx\n", (unsigned long)*database);
}

void
Database::drop(const char *document_id, bool commit)
{
	if (!writable) {
		LOG_ERR(this, "ERROR: database is %s\n", writable ? "w" : "r");
		return;
	}

    document_id  = prefixed(document_id, DOCUMENT_ID_TERM_PREFIX);

	for (int t = 3; t >= 0; --t) {
		LOG(this, "Deleting: -%s- t:%d\n", document_id, t);
		Xapian::WritableDatabase *wdb = static_cast<Xapian::WritableDatabase *>(db);
	    try {
	    	wdb->delete_document(document_id);
	    } catch (const Xapian::Error &e) {
			LOG_ERR(this, "ERROR: %s\n", e.get_msg().c_str());
			if (t) reopen();
			continue;
		}
    	LOG(this, "Document deleted\n");
		if (commit) _commit();
		return;
	}

	LOG_ERR(this, "ERROR: Cannot delete document: %s!\n", document_id);
}

char*
Database::prefixed(const char *term, const char prefix)
{
    char *prefix_term = new char[strlen(term) + 3];
    int i = 0;

    prefix_term[0] = toupper(prefix);
    prefix_term[1] = ':';

    while (term[i] != '\0') {
        prefix_term[i + 2] = term[i];
        i++;
    }
    prefix_term[i + 2] = '\0';
    return prefix_term;
}

char*
Database::prefixed_string(const char *term, const char *prefix)
{
    char *prefix_term = new char[strlen(term) + strlen(prefix) + 1];
    int i = 0, j = 0;

    while (prefix[i] != '\0') {
  	  	prefix_term[i] = toupper(prefix[i]);
  	  	i++;
    }
    while (term[j] != '\0') {
        prefix_term[i] = term[j];
        i++;
        j++;
    }
    prefix_term[i] = '\0';
    return prefix_term;
}

void
Database::_commit()
{
	for (int t = 3; t >= 0; --t) {
		LOG(this, "Commit: t%d\n", t);
		Xapian::WritableDatabase *wdb = static_cast<Xapian::WritableDatabase *>(db);
		try {
			wdb->commit();
		} catch (const Xapian::Error &e) {
			LOG_ERR(this, "ERROR: %s\n", e.get_msg().c_str());
			if (t) reopen();
			continue;
		}

		LOG(this, "Commit made\n");
		return;
	}

	LOG_ERR(this, "ERROR: Cannot do commit!\n");
}

void
Database::index(const char *document, bool commit)
{
	if (!writable) {
		LOG_ERR(this, "ERROR: database is %s\n", writable ? "w" : "r");
		return;
	}

	Xapian::WritableDatabase *wdb = static_cast<Xapian::WritableDatabase *>(db);

	cJSON *root = cJSON_Parse(document);
	if (!root) {
		printf("Error before: [%s]\n",cJSON_GetErrorPtr());
	}
	cJSON *doc_id = root ? cJSON_GetObjectItem(root, "id") : NULL;
    cJSON *document_data = root ? cJSON_GetObjectItem(root, "data") : NULL;
    cJSON *document_values = root ? cJSON_GetObjectItem(root, "values") : NULL;
    cJSON *document_terms = root ? cJSON_GetObjectItem(root, "terms") : NULL;
  	LOG(this, "Document_terms: %s\n", cJSON_Print(document_terms));

    Xapian::Document doc;

    LOG(this, "Start DATA");
    if (document_data) {
    	const char *doc_data = cJSON_Print(document_data);
    	LOG(this, "Document data: %s\n", doc_data);
		doc.set_data(doc_data);
    }
    LOG(this, "End DATA");

    //Document Values

    LOG(this, "Start values");
    if (document_values) {
    	for (int i = 0; i < cJSON_GetArraySize(document_values); i++) {
    		cJSON *value = cJSON_GetArrayItem(document_values, i);
            if (value->type == 3) {
            	doc.add_value((i+1), Xapian::sortable_serialise(value->valuedouble));
            	LOG(this, "%s: %f\n", value->string, value->valuedouble);
            } else {
            	LOG_ERR(this, "ERROR: %s must be a double (%s)\n", value->string, cJSON_Print(value));
            }
  		}
    }
    LOG(this, "End values");

    //Make sure document_id is also a term (otherwise it doesn't replace an existing document)
    const char *document_id  = prefixed(doc_id->valuestring, DOCUMENT_ID_TERM_PREFIX);
    doc.add_value(0, doc_id->valuestring);
    doc.add_boolean_term(document_id);

    //Document Terms
    LOG(this, "Document terms: %s\n", cJSON_Print(document_terms));
    if (document_terms) {
    	LOG(this, "Terms..\n");
    	for (int i = 0; i < cJSON_GetArraySize(document_terms); i++) {
            cJSON *term = cJSON_GetArrayItem(document_terms, i);
            LOG(this, "Prefix: %s   Term: %s\n",term->string, term->valuestring);
            doc.add_term(prefixed_string(term->valuestring, term->string), 1);
            LOG(this, "Term: %s was added\n", prefixed_string(term->valuestring, term->string));
  		}
    }

    cJSON_Delete(root);
}<|endoftext|>


<commit_before>/*
 * This is part of the FL library, a C++ Bayesian filtering library
 * (https://github.com/filtering-library)
 *
 * Copyright (c) 2014 Jan Issac (jan.issac@gmail.com)
 * Copyright (c) 2014 Manuel Wuthrich (manuel.wuthrich@gmail.com)
 *
 * Max-Planck Institute for Intelligent Systems, AMD Lab
 * University of Southern California, CLMC Lab
 *
 * This Source Code Form is subject to the terms of the MIT License (MIT).
 * A copy of the license can be found in the LICENSE file distributed with this
 * source code.
 */

/**
 * \file robust_feature_obsrv_model.hpp
 * \date August 2015
 * \author Jan Issac (jan.issac@gmail.com)
 */

#ifndef FL__MODEL__OBSERVATION__ROBUST_FEATURE_OBSRV_MODEL_HPP
#define FL__MODEL__OBSERVATION__ROBUST_FEATURE_OBSRV_MODEL_HPP

#include <fl/util/meta.hpp>
#include <fl/util/traits.hpp>
#include <fl/util/types.hpp>
#include <fl/util/descriptor.hpp>
#include <fl/distribution/gaussian.hpp>
#include <fl/model/observation/interface/observation_density.hpp>
#include <fl/model/observation/interface/observation_function.hpp>

namespace fl
{

namespace internal
{

enum RobustFeatureDimension { RobustFeatureDimExt = 2 };

}

/**
 * \ingroup observation_models
 *
 * \brief Represents an observation function or model which takes an arbitrary
 *        observation model (one that implements an ObservationFunction & a
 *        an ObservationDensity) as an argument and maps it into a feature space
 *        used by the RobustGaussianFilter.
 */
template <typename ObsrvModel>
class RobustFeatureObsrvModel
    : public ObservationFunction<
                 /* Obsrv type of the size SizeOf<ObsrvModel::Obsrv> + 2 */
                 typename VariateOfSize<
                     JoinSizes<
                         SizeOf<typename ObsrvModel::Obsrv>::Value,
                         internal::RobustFeatureDimExt
                     >::Value
                 >::Type,
                 typename ObsrvModel::State,
                 typename ObsrvModel::Noise>,
      public Descriptor

{
private:
    typedef RobustFeatureObsrvModel<ObsrvModel> This;

public:
    /**
     * \brief \a InputObsrv type which is the same as
     *        ObsrvModel::Obsrv. \a InputObsrv is mapped into the feature space.
     *        The resulting type is \a Obsrv.
     */
    typedef typename ObsrvModel::Obsrv InputObsrv;

    /**
     * \brief \a Obsrv (\f$y_t\f$) type which is a variate of the size
     *        SizeOf<InputObsrv> + 2. \a Obsrv reside in the feature space.
     */
    typedef typename VariateOfSize<
                         JoinSizes<
                             SizeOf<typename ObsrvModel::Obsrv>::Value,
                             internal::RobustFeatureDimExt
                         >::Value
                     >::Type Obsrv;

    /**
     * \brief \a State (\f$x_t\f$) type which is the same as ObsrvModel::State
     */
    typedef typename ObsrvModel::State State;

    /**
     * \brief \a Noise (\f$x_t\f$) type which is the same as ObsrvModel::Noise
     */
    typedef typename ObsrvModel::Noise Noise;

public:
    /**
     * \brief Constructs a robust feature observation model for the robust
     *        gaussian filter
     *
     * \param obsrv_model   Reference to the source observation model
     *
     * \note This model takes only a reference of to an existing lvalue of the
     *       source model
     */
    explicit RobustFeatureObsrvModel(ObsrvModel& obsrv_model)
        : obsrv_model_(obsrv_model)
    { }

    /**
     * \brief Overridable default destructor
     */
    virtual ~RobustFeatureObsrvModel() { }

    /**
     * \brief observation Returns a feature mapped observation
     */
    Obsrv observation(const State& state, const Noise& noise) const override
    {
        Obsrv y = feature_obsrv(obsrv_model_.observation(state, noise), state);
        return y; // RVO
    }

    /**
     * \brief Computes the robust feature given an input feature fron the
     *        source observation model
     */
    virtual Obsrv feature_obsrv(
        const InputObsrv& input_obsrv, const State& state) const
    {
        auto y = Obsrv(obsrv_dimension());

        auto prob_y = body_gaussian_.probability(input_obsrv);
        auto prob_tail = obsrv_model_
                            .tail_model()
                            .probability(input_obsrv, state);

        y(0) = prob_tail;
        if (internal::RobustFeatureDimExt == 2) y(1) = prob_y;
        y.bottomRows(obsrv_model_.obsrv_dimension()) = prob_y * input_obsrv;

        auto weight = obsrv_model_.weight_threshold();
        auto normalizer = (Real(1) - weight) * prob_y + weight * prob_tail;
        y /= normalizer;

        return y;
    }

    /**
     * \brief Sets the feature function (feature observation modek) and
     *        parameters
     * \param body_gaussian     \f${\cal N}(y_t\mid \mu_{y}, \Sigma_{yy})\f$
     * \param mean_state        \f$ \mu_x \f$
     *
     * PAPER REF
     */
    virtual void parameters(
        const State& mean_state,
        const typename FirstMomentOf<InputObsrv>::Type& mean_obsrv,
        const typename SecondMomentOf<InputObsrv>::Type& cov_obsrv)
    {
        mean_state_ = mean_state;
        body_gaussian_.mean(mean_obsrv);
        body_gaussian_.covariance(cov_obsrv);
    }

    /**
     * \brief Returns the dimension of the \a Obsrv which is dim(\a Obsrv) + 2
     */
    int obsrv_dimension() const override
    {
        return obsrv_model_.obsrv_dimension() + internal::RobustFeatureDimExt;
    }

    int noise_dimension() const override
    {
        return obsrv_model_.noise_dimension();
    }

    int state_dimension() const override
    {
        return obsrv_model_.state_dimension();
    }

    ObsrvModel& embedded_obsrv_model()
    {
        return obsrv_model_;
    }

    const ObsrvModel& embedded_obsrv_model() const
    {
        return obsrv_model_;
    }

    virtual std::string name() const
    {
        return "RobustFeatureObsrvModel<"
                + this->list_arguments(embedded_obsrv_model().name())
                + ">";
    }

    virtual std::string description() const
    {
        return "Robust feature observation model with "
                + this->list_descriptions(embedded_obsrv_model().description());
    }

protected:
    /** \cond internal */

    /**
     * \brief Reference to the ource observation model
     */
    ObsrvModel& obsrv_model_;

    /**
     * \brief \f${\cal N}(y_t\mid \mu_{y}, \Sigma_{yy})\f$
     */
    Gaussian<InputObsrv> body_gaussian_;

    /**
     * \brief \f$\mu_x\f$
     */
    State mean_state_;

    /* \endcond */
};

}

#endif
<commit_msg>Factorized parameters into mean_state and body_moments. Fixed body_gaussian_ initialization for dynamics sizes<commit_after>/*
 * This is part of the FL library, a C++ Bayesian filtering library
 * (https://github.com/filtering-library)
 *
 * Copyright (c) 2014 Jan Issac (jan.issac@gmail.com)
 * Copyright (c) 2014 Manuel Wuthrich (manuel.wuthrich@gmail.com)
 *
 * Max-Planck Institute for Intelligent Systems, AMD Lab
 * University of Southern California, CLMC Lab
 *
 * This Source Code Form is subject to the terms of the MIT License (MIT).
 * A copy of the license can be found in the LICENSE file distributed with this
 * source code.
 */

/**
 * \file robust_feature_obsrv_model.hpp
 * \date August 2015
 * \author Jan Issac (jan.issac@gmail.com)
 */

#pragma once

#include <fl/util/meta.hpp>
#include <fl/util/traits.hpp>
#include <fl/util/types.hpp>
#include <fl/util/descriptor.hpp>
#include <fl/distribution/gaussian.hpp>
#include <fl/model/observation/interface/observation_density.hpp>
#include <fl/model/observation/interface/observation_function.hpp>

namespace fl
{

namespace internal
{

enum RobustFeatureDimension { RobustFeatureDimExt = 2 };

}

/**
 * \ingroup observation_models
 *
 * \brief Represents an observation function or model which takes an arbitrary
 *        observation model (one that implements an ObservationFunction & a
 *        an ObservationDensity) as an argument and maps it into a feature space
 *        used by the RobustGaussianFilter.
 */
template <typename ObsrvModel>
class RobustFeatureObsrvModel
    : public ObservationFunction<
                 /* Obsrv type of the size SizeOf<ObsrvModel::Obsrv> + 2 */
                 typename VariateOfSize<
                     JoinSizes<
                         SizeOf<typename ObsrvModel::Obsrv>::Value,
                         internal::RobustFeatureDimExt
                     >::Value
                 >::Type,
                 typename ObsrvModel::State,
                 typename ObsrvModel::Noise>,
      public Descriptor

{
private:
    typedef RobustFeatureObsrvModel<ObsrvModel> This;

public:
    /**
     * \brief \a InputObsrv type which is the same as
     *        ObsrvModel::Obsrv. \a InputObsrv is mapped into the feature space.
     *        The resulting type is \a Obsrv.
     */
    typedef typename ObsrvModel::Obsrv InputObsrv;

    /**
     * \brief \a Obsrv (\f$y_t\f$) type which is a variate of the size
     *        SizeOf<InputObsrv> + 2. \a Obsrv reside in the feature space.
     */
    typedef typename VariateOfSize<
                         JoinSizes<
                             SizeOf<typename ObsrvModel::Obsrv>::Value,
                             internal::RobustFeatureDimExt
                         >::Value
                     >::Type Obsrv;

    /**
     * \brief \a State (\f$x_t\f$) type which is the same as ObsrvModel::State
     */
    typedef typename ObsrvModel::State State;

    /**
     * \brief \a Noise (\f$x_t\f$) type which is the same as ObsrvModel::Noise
     */
    typedef typename ObsrvModel::Noise Noise;

public:
    /**
     * \brief Constructs a robust feature observation model for the robust
     *        gaussian filter
     *
     * \param obsrv_model   Reference to the source observation model
     *
     * \note This model takes only a reference of to an existing lvalue of the
     *       source model
     */
    explicit RobustFeatureObsrvModel(ObsrvModel& obsrv_model)
        : obsrv_model_(obsrv_model),
          body_gaussian_(obsrv_model.obsrv_dimension())
    { }

    /**
     * \brief Overridable default destructor
     */
    virtual ~RobustFeatureObsrvModel() { }

    /**
     * \brief observation Returns a feature mapped observation
     */
    Obsrv observation(const State& state, const Noise& noise) const override
    {
        Obsrv y = feature_obsrv(obsrv_model_.observation(state, noise));
        return y; // RVO
    }

    /**
     * \brief Computes the robust feature given an input feature fron the
     *        source observation model
     */
    virtual Obsrv feature_obsrv(const InputObsrv& input_obsrv) const
    {
        auto y = Obsrv(obsrv_dimension());

        auto prob_y = body_gaussian_.probability(input_obsrv);
        auto prob_tail = obsrv_model_
                            .tail_model()
                            .probability(input_obsrv, mean_state_);

        y(0) = prob_tail;
        if (internal::RobustFeatureDimExt == 2) y(1) = prob_y;
        y.bottomRows(obsrv_model_.obsrv_dimension()) = prob_y * input_obsrv;

        auto weight = obsrv_model_.weight_threshold();
        auto normalizer = (Real(1) - weight) * prob_y + weight * prob_tail;
        y /= normalizer;

        return y;
    }

//    /**
//     * \brief Sets the feature function (feature observation modek) and
//     *        parameters
//     * \param body_gaussian     \f${\cal N}(y_t\mid \mu_{y}, \Sigma_{yy})\f$
//     * \param mean_state        \f$ \mu_x \f$
//     *
//     * PAPER REF
//     */
//    virtual void parameters(
//        const State& mean_state,
//        const typename FirstMomentOf<InputObsrv>::Type& mean_obsrv,
//        const typename SecondMomentOf<InputObsrv>::Type& cov_obsrv)
//    {
//        mean_state_ = mean_state;
//        body_gaussian_.mean(mean_obsrv);
//        body_gaussian_.covariance(cov_obsrv);
//    }

    /**
     * \brief Sets the feature function \a mean_state;
     * \param mean_state        \f$ \mu_x \f$
     *
     * \todo PAPER REF
     */
    virtual void mean_state(const State& mean_state)
    {
        mean_state_ = mean_state;
    }

    /**
     * \brief Sets the feature function body moments
     * \param body_gaussian     \f${\cal N}(y_t\mid \mu_{y}, \Sigma_{yy})\f$
     *
     * \todo PAPER REF
     */
    virtual void body_moments(
        const typename FirstMomentOf<InputObsrv>::Type& mean_obsrv,
        const typename SecondMomentOf<InputObsrv>::Type& cov_obsrv)
    {
        body_gaussian_.mean(mean_obsrv);
        body_gaussian_.covariance(cov_obsrv);
    }

    /**
     * \brief Returns the dimension of the \a Obsrv which is dim(\a Obsrv) + 2
     */
    int obsrv_dimension() const override
    {
        return obsrv_model_.obsrv_dimension() + internal::RobustFeatureDimExt;
    }

    int noise_dimension() const override
    {
        return obsrv_model_.noise_dimension();
    }

    int state_dimension() const override
    {
        return obsrv_model_.state_dimension();
    }

    ObsrvModel& embedded_obsrv_model()
    {
        return obsrv_model_;
    }

    const ObsrvModel& embedded_obsrv_model() const
    {
        return obsrv_model_;
    }

    virtual std::string name() const
    {
        return "RobustFeatureObsrvModel<"
                + this->list_arguments(embedded_obsrv_model().name())
                + ">";
    }

    virtual std::string description() const
    {
        return "Robust feature observation model with "
                + this->list_descriptions(embedded_obsrv_model().description());
    }

protected:
    /** \cond internal */

    /**
     * \brief Reference to the ource observation model
     */
    ObsrvModel& obsrv_model_;

    /**
     * \brief \f${\cal N}(y_t\mid \mu_{y}, \Sigma_{yy})\f$
     */
    Gaussian<InputObsrv> body_gaussian_;

    /**
     * \brief \f$\mu_x\f$
     */
    State mean_state_;

    /* \endcond */
};

}

<|endoftext|>


<commit_before>//=======================================================================
// Copyright Baptiste Wicht 2015-2017.
// Distributed under the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <fstream>
#include <sstream>

#include <dirent.h>

#include "dataset.hpp"
#include "config.hpp"
#include "utils.hpp"

namespace {

void read_word_labels(spot_dataset& dataset, const std::string& path) {
    std::ifstream word_labels_stream(path + "/ground_truth/word_labels.txt");

    while (!word_labels_stream.eof()) {
        std::string image_name;
        word_labels_stream >> image_name;

        std::string label;
        word_labels_stream >> label;

        if (image_name.empty() || label.empty()) {
            continue;
        }

        std::istringstream ss(label);
        std::string token;

        while (std::getline(ss, token, '-')) {
            dataset.word_labels[image_name].push_back(token);
        }
    }
}

void read_word_labels_ak(spot_dataset& dataset, const std::string& path) {
    std::ifstream word_labels_stream(path + "/ground_truth/word_labels.txt");

    while (!word_labels_stream.eof()) {
        std::string image_name;
        word_labels_stream >> image_name;

        std::string label;
        word_labels_stream >> label;

        if (image_name.empty() || label.empty()) {
            continue;
        }

        // Get rid of AK crap
        if (label == "N/A" || label == "." || label == "," || label == "\"" || label == "-") {
            continue;
        }

        std::transform(label.begin(), label.end(), label.begin(), ::tolower);

        for(char c : label){
            dataset.word_labels[image_name].push_back(std::string() + c);
        }
    }
}

void read_line_transcriptions(spot_dataset& dataset, const std::string& path) {
    std::ifstream line_transcriptions_stream(path + "/ground_truth/transcription.txt");

    while (!line_transcriptions_stream.eof()) {
        std::string image_name;
        line_transcriptions_stream >> image_name;

        std::string label;
        line_transcriptions_stream >> label;

        if (image_name.empty() || label.empty()) {
            continue;
        }

        std::istringstream ss(label);
        std::string word_token;

        while (std::getline(ss, word_token, '|')) {
            std::vector<std::string> word_labels;

            std::istringstream ss2(word_token);
            std::string token;

            while (std::getline(ss2, token, '-')) {
                word_labels.push_back(token);
            }

            dataset.line_transcriptions[image_name].emplace_back(std::move(word_labels));
        }
    }
}

void read_images(std::unordered_map<std::string, cv::Mat>& map, const std::string& file_path) {
    std::cout << "Read images from '" << file_path << "'" << std::endl;

    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 3 || file_name.find(".png") != file_name.size() - 4) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        map[file_name] = cv::imread(full_name, CV_LOAD_IMAGE_ANYDEPTH);

        if (!map[file_name].data) {
            std::cout << "Impossible to read image " << full_name << std::endl;
        }
    }
}

void read_images_ak(std::unordered_map<std::string, cv::Mat>& map, const std::string& file_path, const std::string& sub) {
    std::cout << "Read images from '" << file_path << "/" << sub << "'" << std::endl;

    auto full_path = file_path + "/" + sub;

    struct dirent* entry;
    auto dir = opendir(full_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 3 || file_name.find(".png") != file_name.size() - 4) {
            continue;
        }

        std::string full_name(full_path + "/" + file_name);

        auto key = sub + "/" + file_name;

        map[key] = cv::imread(full_name, CV_LOAD_IMAGE_ANYDEPTH);

        if (!map[key].data) {
            std::cout << "Impossible to read image " << full_name << std::endl;
        }
    }
}

void read_line_images(spot_dataset& dataset, const std::string& path) {
    read_images(dataset.line_images, path + "/data/line_images_normalized/");
}

void read_word_images(const config& /*conf*/, spot_dataset& dataset, const std::string& path) {
    read_images(dataset.word_images, path + "/data/word_images_normalized/");
}

void read_word_images_gw(const config& conf, spot_dataset& dataset, const std::string& path) {
    if(conf.gray){
        read_images(dataset.word_images, path + "/data/word_gray/");
    } else if(conf.binary){
        read_images(dataset.word_images, path + "/data/word_binary/");
    } else {
        read_images(dataset.word_images, path + "/data/word_images_normalized/");
    }
}

void read_word_images_ak(const config& conf, spot_dataset& dataset, const std::string& path) {
    read_images_ak(dataset.word_images, path + "/data/word_images_normalized", "test");
    read_images_ak(dataset.word_images, path + "/data/word_images_normalized", "train");
}

void read_list(std::vector<std::string>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string image_name;
        stream >> image_name;

        if (!image_name.empty()) {
            list.push_back(image_name);
        }
    }
}

void read_keywords(std::vector<std::vector<std::string>>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string keywords;
        stream >> keywords;

        if (!keywords.empty()) {
            list.emplace_back();

            std::istringstream ss(keywords);
            std::string token;

            while (std::getline(ss, token, '-')) {
                list.back().push_back(token);
            }
        }
    }
}

void read_keywords_iam(std::vector<std::vector<std::string>>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string line_keywords;
        std::getline(stream, line_keywords);

        if(line_keywords.empty()){
            continue;
        }

        std::string keywords(line_keywords.begin(), line_keywords.begin() + line_keywords.find(' '));

        if(keywords.empty()){
            continue;
        }

        list.emplace_back();

        std::istringstream ss(keywords);
        std::string token;

        while (std::getline(ss, token, '-')) {
            list.back().push_back(token);
        }
    }
}

void load_sets_washington(spot_dataset& dataset, const std::string& path) {
    std::string file_path(path + "/sets");

    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 2) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        read_list(dataset.sets[file_name].test_set, full_name + "/test.txt");
        read_list(dataset.sets[file_name].train_set, full_name + "/train.txt");
        read_list(dataset.sets[file_name].validation_set, full_name + "/valid.txt");
        read_keywords(dataset.sets[file_name].keywords, full_name + "/keywords.txt");
    }
}

void load_sets_ak(spot_dataset& dataset, const std::string& path) {
    std::string set_name = "cv1";

    auto& test_set  = dataset.sets[set_name].test_set;
    auto& train_set = dataset.sets[set_name].train_set;
    auto& valid_set = dataset.sets[set_name].validation_set;

    for(auto& image : dataset.word_images){
        if(image.first.substr(0, 5) == "train"){
            train_set.push_back(image.first);
        } else if(image.first.substr(0, 4) == "test"){
            test_set.push_back(image.first);
        }
    }

    valid_set.clear(); // Nothing
}

void load_sets_parzival(spot_dataset& dataset, const std::string& path) {
    std::string full_name(path + "/sets1/");

    read_list(dataset.sets["cv1"].test_set, full_name + "/test.txt");
    read_list(dataset.sets["cv1"].train_set, full_name + "/train.txt");
    read_list(dataset.sets["cv1"].validation_set, full_name + "/valid.txt");
    read_keywords(dataset.sets["cv1"].keywords, full_name + "/keywords.txt");
}

void load_sets_iam(spot_dataset& dataset, const std::string& path) {
    std::string file_path(path + "/sets");

    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 2) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        read_list(dataset.sets[file_name].test_set, full_name + "/test.txt");
        read_list(dataset.sets[file_name].train_set, full_name + "/train.txt");
        read_list(dataset.sets[file_name].validation_set, full_name + "/valid.txt");
        read_keywords_iam(dataset.sets[file_name].keywords, full_name + "/keywords.txt");
    }
}

} //end of anonymous namespace

spot_dataset read_washington(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images_gw(conf, dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_washington(dataset, path);

    return dataset;
}

spot_dataset read_ak(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_labels_ak(dataset, path);
    read_word_images_ak(conf, dataset, path);

    // Now we need to filter the crap out of the data set

    auto it = dataset.word_images.begin();

    while (it != dataset.word_images.end()) {
        if (!dataset.word_labels.count({it->first.begin(), it->first.end() - 4})) {
            it = dataset.word_images.erase(it);
        } else {
            ++it;
        }
    }

    load_sets_ak(dataset, path);

    return dataset;
}

spot_dataset read_botany(const config& conf, const std::string& path) {
    // Seems the same crap
    return read_ak(conf, path);
}

spot_dataset read_manmatha(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_images(conf, dataset, path);

    load_sets_washington(dataset, path);

    return dataset;
}

spot_dataset read_parzival(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images(conf, dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_parzival(dataset, path);

    return dataset;
}

spot_dataset read_iam(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images(conf, dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_iam(dataset, path);

    return dataset;
}

std::vector<std::vector<std::string>> select_keywords(const config& conf, const spot_dataset& dataset, const spot_dataset_set& set, names train_word_names, names test_image_names, bool verbose) {
    std::vector<std::vector<std::string>> keywords;

    if(conf.ak || conf.botany){
        std::set<std::vector<std::string>> base_keywords;

        for (auto& labels : dataset.word_labels) {
            base_keywords.insert(labels.second);
        }

        for (auto& keyword : base_keywords) {
            bool found = false;

            for (auto& labels : dataset.word_labels) {
                if (keyword == labels.second && std::find(train_word_names.begin(), train_word_names.end(), labels.first) != train_word_names.end()) {
                    found = true;
                    break;
                }
            }

            if (found) {
                auto total_test = std::count_if(test_image_names.begin(), test_image_names.end(),
                                                [&dataset, &keyword](auto& i) {
                                                    std::string key{i.begin(), i.end() - 4};
                                                    return dataset.word_labels.at(key) == keyword;
                                                });

                if (total_test > 0) {
                    keywords.push_back(keyword);
                }
            }
        }
    } else {
        for (auto& keyword : set.keywords) {
            bool found = false;

            for (auto& labels : dataset.word_labels) {
                if (keyword == labels.second && std::find(train_word_names.begin(), train_word_names.end(), labels.first) != train_word_names.end()) {
                    found = true;
                    break;
                }
            }

            if (found) {
                auto total_test = std::count_if(test_image_names.begin(), test_image_names.end(),
                                                [&dataset, &keyword](auto& i) { return dataset.word_labels.at({i.begin(), i.end() - 4}) == keyword; });

                if (total_test > 0) {
                    keywords.push_back(keyword);
                }
            }
        }
    }

    if(verbose){
         std::cout << "Selected " << keywords.size() << " keyword out of " << set.keywords.size() << std::endl;
    }

    return keywords;
}
<commit_msg>Resize for patches<commit_after>//=======================================================================
// Copyright Baptiste Wicht 2015-2017.
// Distributed under the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <fstream>
#include <sstream>

#include <dirent.h>

#include "dataset.hpp"
#include "config.hpp"
#include "utils.hpp"

namespace {

void read_word_labels(spot_dataset& dataset, const std::string& path) {
    std::ifstream word_labels_stream(path + "/ground_truth/word_labels.txt");

    while (!word_labels_stream.eof()) {
        std::string image_name;
        word_labels_stream >> image_name;

        std::string label;
        word_labels_stream >> label;

        if (image_name.empty() || label.empty()) {
            continue;
        }

        std::istringstream ss(label);
        std::string token;

        while (std::getline(ss, token, '-')) {
            dataset.word_labels[image_name].push_back(token);
        }
    }
}

void read_word_labels_ak(spot_dataset& dataset, const std::string& path) {
    std::ifstream word_labels_stream(path + "/ground_truth/word_labels.txt");

    while (!word_labels_stream.eof()) {
        std::string image_name;
        word_labels_stream >> image_name;

        std::string label;
        word_labels_stream >> label;

        if (image_name.empty() || label.empty()) {
            continue;
        }

        // Get rid of AK crap
        if (label == "N/A" || label == "." || label == "," || label == "\"" || label == "-") {
            continue;
        }

        std::transform(label.begin(), label.end(), label.begin(), ::tolower);

        for(char c : label){
            dataset.word_labels[image_name].push_back(std::string() + c);
        }
    }
}

void read_line_transcriptions(spot_dataset& dataset, const std::string& path) {
    std::ifstream line_transcriptions_stream(path + "/ground_truth/transcription.txt");

    while (!line_transcriptions_stream.eof()) {
        std::string image_name;
        line_transcriptions_stream >> image_name;

        std::string label;
        line_transcriptions_stream >> label;

        if (image_name.empty() || label.empty()) {
            continue;
        }

        std::istringstream ss(label);
        std::string word_token;

        while (std::getline(ss, word_token, '|')) {
            std::vector<std::string> word_labels;

            std::istringstream ss2(word_token);
            std::string token;

            while (std::getline(ss2, token, '-')) {
                word_labels.push_back(token);
            }

            dataset.line_transcriptions[image_name].emplace_back(std::move(word_labels));
        }
    }
}

void read_images(std::unordered_map<std::string, cv::Mat>& map, const std::string& file_path) {
    std::cout << "Read images from '" << file_path << "'" << std::endl;

    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 3 || file_name.find(".png") != file_name.size() - 4) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        map[file_name] = cv::imread(full_name, CV_LOAD_IMAGE_ANYDEPTH);

        if (!map[file_name].data) {
            std::cout << "Impossible to read image " << full_name << std::endl;
        }
    }
}

void read_images_ak(const config& conf, std::unordered_map<std::string, cv::Mat>& map, const std::string& file_path, const std::string& sub) {
    std::cout << "Read images from '" << file_path << "/" << sub << "'" << std::endl;

    auto full_path = file_path + "/" + sub;

    struct dirent* entry;
    auto dir = opendir(full_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 3 || file_name.find(".png") != file_name.size() - 4) {
            continue;
        }

        std::string full_name(full_path + "/" + file_name);

        auto key = sub + "/" + file_name;

        cv::Mat base_image = cv::imread(full_name, CV_LOAD_IMAGE_ANYDEPTH);

        if (conf.method == Method::Patches) {
            if (base_image.size().height == HEIGHT) {
                map[key] = base_image;
            } else {
                float ratio = float(HEIGHT) / base_image.size().height;

                cv::Mat scaled_normalized(
                    cv::Size(static_cast<size_t>(base_image.size().width * ratio), HEIGHT),
                    CV_8U);
                cv::resize(base_image, scaled_normalized, scaled_normalized.size(), cv::INTER_AREA);
                cv::adaptiveThreshold(scaled_normalized, map[key], 255, CV_ADAPTIVE_THRESH_MEAN_C, CV_THRESH_BINARY, 7, 2);
            }
        } else {
            map[key] = base_image;
        }

        if (!map[key].data) {
            std::cout << "Impossible to read image " << full_name << std::endl;
        }
    }
}

void read_line_images(spot_dataset& dataset, const std::string& path) {
    read_images(dataset.line_images, path + "/data/line_images_normalized/");
}

void read_word_images(const config& /*conf*/, spot_dataset& dataset, const std::string& path) {
    read_images(dataset.word_images, path + "/data/word_images_normalized/");
}

void read_word_images_gw(const config& conf, spot_dataset& dataset, const std::string& path) {
    if(conf.gray){
        read_images(dataset.word_images, path + "/data/word_gray/");
    } else if(conf.binary){
        read_images(dataset.word_images, path + "/data/word_binary/");
    } else {
        read_images(dataset.word_images, path + "/data/word_images_normalized/");
    }
}

void read_word_images_ak(const config& conf, spot_dataset& dataset, const std::string& path) {
    read_images_ak(conf, dataset.word_images, path + "/data/word_images_normalized", "test");
    read_images_ak(conf, dataset.word_images, path + "/data/word_images_normalized", "train");
}

void read_list(std::vector<std::string>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string image_name;
        stream >> image_name;

        if (!image_name.empty()) {
            list.push_back(image_name);
        }
    }
}

void read_keywords(std::vector<std::vector<std::string>>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string keywords;
        stream >> keywords;

        if (!keywords.empty()) {
            list.emplace_back();

            std::istringstream ss(keywords);
            std::string token;

            while (std::getline(ss, token, '-')) {
                list.back().push_back(token);
            }
        }
    }
}

void read_keywords_iam(std::vector<std::vector<std::string>>& list, const std::string& path) {
    std::ifstream stream(path);

    while (!stream.eof()) {
        std::string line_keywords;
        std::getline(stream, line_keywords);

        if(line_keywords.empty()){
            continue;
        }

        std::string keywords(line_keywords.begin(), line_keywords.begin() + line_keywords.find(' '));

        if(keywords.empty()){
            continue;
        }

        list.emplace_back();

        std::istringstream ss(keywords);
        std::string token;

        while (std::getline(ss, token, '-')) {
            list.back().push_back(token);
        }
    }
}

void load_sets_washington(spot_dataset& dataset, const std::string& path) {
    std::string file_path(path + "/sets");

    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 2) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        read_list(dataset.sets[file_name].test_set, full_name + "/test.txt");
        read_list(dataset.sets[file_name].train_set, full_name + "/train.txt");
        read_list(dataset.sets[file_name].validation_set, full_name + "/valid.txt");
        read_keywords(dataset.sets[file_name].keywords, full_name + "/keywords.txt");
    }
}

void load_sets_ak(spot_dataset& dataset, const std::string& path) {
    std::string set_name = "cv1";

    auto& test_set  = dataset.sets[set_name].test_set;
    auto& train_set = dataset.sets[set_name].train_set;
    auto& valid_set = dataset.sets[set_name].validation_set;

    for(auto& image : dataset.word_images){
        if(image.first.substr(0, 5) == "train"){
            train_set.push_back(image.first);
        } else if(image.first.substr(0, 4) == "test"){
            test_set.push_back(image.first);
        }
    }

    valid_set.clear(); // Nothing
}

void load_sets_parzival(spot_dataset& dataset, const std::string& path) {
    std::string full_name(path + "/sets1/");

    read_list(dataset.sets["cv1"].test_set, full_name + "/test.txt");
    read_list(dataset.sets["cv1"].train_set, full_name + "/train.txt");
    read_list(dataset.sets["cv1"].validation_set, full_name + "/valid.txt");
    read_keywords(dataset.sets["cv1"].keywords, full_name + "/keywords.txt");
}

void load_sets_iam(spot_dataset& dataset, const std::string& path) {
    std::string file_path(path + "/sets");

    struct dirent* entry;
    auto dir = opendir(file_path.c_str());

    while ((entry = readdir(dir))) {
        std::string file_name(entry->d_name);

        if (file_name.size() <= 2) {
            continue;
        }

        std::string full_name(file_path + "/" + file_name);

        read_list(dataset.sets[file_name].test_set, full_name + "/test.txt");
        read_list(dataset.sets[file_name].train_set, full_name + "/train.txt");
        read_list(dataset.sets[file_name].validation_set, full_name + "/valid.txt");
        read_keywords_iam(dataset.sets[file_name].keywords, full_name + "/keywords.txt");
    }
}

} //end of anonymous namespace

spot_dataset read_washington(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images_gw(conf, dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_washington(dataset, path);

    return dataset;
}

spot_dataset read_ak(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_labels_ak(dataset, path);
    read_word_images_ak(conf, dataset, path);

    // Now we need to filter the crap out of the data set

    auto it = dataset.word_images.begin();

    while (it != dataset.word_images.end()) {
        if (!dataset.word_labels.count({it->first.begin(), it->first.end() - 4})) {
            it = dataset.word_images.erase(it);
        } else {
            ++it;
        }
    }

    load_sets_ak(dataset, path);

    return dataset;
}

spot_dataset read_botany(const config& conf, const std::string& path) {
    // Seems the same crap
    return read_ak(conf, path);
}

spot_dataset read_manmatha(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_images(conf, dataset, path);

    load_sets_washington(dataset, path);

    return dataset;
}

spot_dataset read_parzival(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images(conf, dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_parzival(dataset, path);

    return dataset;
}

spot_dataset read_iam(const config& conf, const std::string& path) {
    spot_dataset dataset;

    read_word_labels(dataset, path);
    read_word_images(conf, dataset, path);

    if(dataset_read_lines){
        read_line_transcriptions(dataset, path);
        read_line_images(dataset, path);
    }

    load_sets_iam(dataset, path);

    return dataset;
}

std::vector<std::vector<std::string>> select_keywords(const config& conf, const spot_dataset& dataset, const spot_dataset_set& set, names train_word_names, names test_image_names, bool verbose) {
    std::vector<std::vector<std::string>> keywords;

    if(conf.ak || conf.botany){
        std::set<std::vector<std::string>> base_keywords;

        for (auto& labels : dataset.word_labels) {
            base_keywords.insert(labels.second);
        }

        for (auto& keyword : base_keywords) {
            bool found = false;

            for (auto& labels : dataset.word_labels) {
                if (keyword == labels.second && std::find(train_word_names.begin(), train_word_names.end(), labels.first) != train_word_names.end()) {
                    found = true;
                    break;
                }
            }

            if (found) {
                auto total_test = std::count_if(test_image_names.begin(), test_image_names.end(),
                                                [&dataset, &keyword](auto& i) {
                                                    std::string key{i.begin(), i.end() - 4};
                                                    return dataset.word_labels.at(key) == keyword;
                                                });

                if (total_test > 0) {
                    keywords.push_back(keyword);
                }
            }
        }
    } else {
        for (auto& keyword : set.keywords) {
            bool found = false;

            for (auto& labels : dataset.word_labels) {
                if (keyword == labels.second && std::find(train_word_names.begin(), train_word_names.end(), labels.first) != train_word_names.end()) {
                    found = true;
                    break;
                }
            }

            if (found) {
                auto total_test = std::count_if(test_image_names.begin(), test_image_names.end(),
                                                [&dataset, &keyword](auto& i) { return dataset.word_labels.at({i.begin(), i.end() - 4}) == keyword; });

                if (total_test > 0) {
                    keywords.push_back(keyword);
                }
            }
        }
    }

    if(verbose){
         std::cout << "Selected " << keywords.size() << " keyword out of " << set.keywords.size() << std::endl;
    }

    return keywords;
}
<|endoftext|>


<commit_before>//===- lib/ReaderWriter/ELF/WriterELF.cpp ---------------------------------===//
//
//                             The LLVM Linker
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#include "lld/ReaderWriter/Writer.h"

#include "DefaultLayout.h"
#include "TargetLayout.h"
#include "ExecutableAtoms.h"

#include "lld/ReaderWriter/ELFTargetInfo.h"

using namespace llvm;
using namespace llvm::object;
namespace lld {
namespace elf {
template<class ELFT>
class ExecutableWriter;

//===----------------------------------------------------------------------===//
//  ExecutableWriter Class
//===----------------------------------------------------------------------===//
template<class ELFT>
class ExecutableWriter : public ELFWriter {
public:
  typedef Elf_Shdr_Impl<ELFT> Elf_Shdr;
  typedef Elf_Sym_Impl<ELFT> Elf_Sym;

  ExecutableWriter(const ELFTargetInfo &ti);

private:
  // build the sections that need to be created
  void buildChunks(const File &file);
  virtual error_code writeFile(const File &File, StringRef path);
  void buildAtomToAddressMap();
  void buildSymbolTable ();
  void buildSectionHeaderTable();
  void assignSectionsWithNoSegments();
  void addAbsoluteUndefinedSymbols(const File &File);
  void addDefaultAtoms();
  void addFiles(InputFiles&);
  void finalizeDefaultAtomValues();

  uint64_t addressOfAtom(const Atom *atom) {
    return _atomToAddressMap[atom];
  }

  void createDefaultSections();

  const ELFTargetInfo &_targetInfo;
  TargetHandler<ELFT> &_targetHandler;

  typedef llvm::DenseMap<const Atom *, uint64_t> AtomToAddress;
  AtomToAddress _atomToAddressMap;
  llvm::BumpPtrAllocator _chunkAllocate;
  TargetLayout<ELFT> *_layout;
  Header<ELFT> *_Header;
  ProgramHeader<ELFT> *_programHeader;
  SymbolTable<ELFT> * _symtab;
  StringTable<ELFT> *_strtab;
  StringTable<ELFT> *_shstrtab;
  SectionHeader<ELFT> *_shdrtab;
  CRuntimeFile<ELFT> _runtimeFile;
};

//===----------------------------------------------------------------------===//
//  ExecutableWriter
//===----------------------------------------------------------------------===//
template <class ELFT>
ExecutableWriter<ELFT>::ExecutableWriter(const ELFTargetInfo &ti)
    : _targetInfo(ti), _targetHandler(ti.getTargetHandler<ELFT>()),
      _runtimeFile(ti) {
  _layout = &_targetHandler.targetLayout();
}

template <class ELFT>
void ExecutableWriter<ELFT>::buildChunks(const File &file) {
  for (const DefinedAtom *definedAtom : file.defined() ) {
    _layout->addAtom(definedAtom);
  }
  /// Add all the absolute atoms to the layout
  for (const AbsoluteAtom *absoluteAtom : file.absolute()) {
    _layout->addAtom(absoluteAtom);
  }
}

template<class ELFT>
void ExecutableWriter<ELFT>::buildSymbolTable () {
  for (auto sec : _layout->sections())
    if (auto section = dyn_cast<Section<ELFT>>(sec))
      for (const auto &atom : section->atoms())
        _symtab->addSymbol(atom->_atom, section->ordinal(), atom->_virtualAddr);
}

template<class ELFT>
void
ExecutableWriter<ELFT>::addAbsoluteUndefinedSymbols(const File &file) {
  // add all the absolute symbols that the layout contains to the output symbol
  // table
  for (auto &atom : _layout->absoluteAtoms())
    _symtab->addSymbol(atom->_atom, ELF::SHN_ABS, atom->_virtualAddr);
  for (const UndefinedAtom *a : file.undefined())
    _symtab->addSymbol(a, ELF::SHN_UNDEF);
}

template<class ELFT>
void ExecutableWriter<ELFT>::buildAtomToAddressMap () {
  for (auto sec : _layout->sections())
    if (auto section = dyn_cast<Section<ELFT>>(sec))
      for (const auto &atom : section->atoms())
        _atomToAddressMap[atom->_atom] = atom->_virtualAddr;
  // build the atomToAddressMap that contains absolute symbols too
  for (auto &atom : _layout->absoluteAtoms())
    _atomToAddressMap[atom->_atom] = atom->_virtualAddr;
}

template<class ELFT>
void ExecutableWriter<ELFT>::buildSectionHeaderTable() {
  for (auto mergedSec : _layout->mergedSections()) {
    if (mergedSec->kind() != Chunk<ELFT>::K_ELFSection)
      continue;
    if (mergedSec->hasSegment())
      _shdrtab->appendSection(mergedSec);
  }
}

template<class ELFT>
void ExecutableWriter<ELFT>::assignSectionsWithNoSegments() {
  for (auto mergedSec : _layout->mergedSections()) {
    if (mergedSec->kind() != Chunk<ELFT>::K_ELFSection)
      continue;
    if (!mergedSec->hasSegment())
      _shdrtab->appendSection(mergedSec);
  }
  _layout->assignOffsetsForMiscSections();
  for (auto sec : _layout->sections())
    if (auto section = dyn_cast<Section<ELFT>>(sec))
      if (!DefaultLayout<ELFT>::hasOutputSegment(section))
        _shdrtab->updateSection(section);
}

/// \brief Add absolute symbols by default. These are linker added
/// absolute symbols
template<class ELFT>
void ExecutableWriter<ELFT>::addDefaultAtoms() {
  _runtimeFile.addUndefinedAtom(_targetInfo.getEntry());
  _runtimeFile.addAbsoluteAtom("__bss_start");
  _runtimeFile.addAbsoluteAtom("__bss_end");
  _runtimeFile.addAbsoluteAtom("_end");
  _runtimeFile.addAbsoluteAtom("end");
  _runtimeFile.addAbsoluteAtom("__init_array_start");
  _runtimeFile.addAbsoluteAtom("__init_array_end");
  _runtimeFile.addAbsoluteAtom("__rela_iplt_start");
  _runtimeFile.addAbsoluteAtom("__rela_iplt_end");
}

/// \brief Hook in lld to add CRuntime file
template <class ELFT>
void ExecutableWriter<ELFT>::addFiles(InputFiles &inputFiles) {
  addDefaultAtoms();
  inputFiles.prependFile(_runtimeFile);
  // Give a chance for the target to add atoms
  _targetHandler.addFiles(inputFiles);
}

/// Finalize the value of all the absolute symbols that we
/// created
template<class ELFT>
void ExecutableWriter<ELFT>::finalizeDefaultAtomValues() {
  auto bssStartAtomIter = _layout->findAbsoluteAtom("__bss_start");
  auto bssEndAtomIter = _layout->findAbsoluteAtom("__bss_end");
  auto underScoreEndAtomIter = _layout->findAbsoluteAtom("_end");
  auto endAtomIter = _layout->findAbsoluteAtom("end");
  auto initArrayStartIter = _layout->findAbsoluteAtom("__init_array_start");
  auto initArrayEndIter = _layout->findAbsoluteAtom("__init_array_end");
  auto realIpltStartIter = _layout->findAbsoluteAtom("__rela_iplt_start");
  auto realIpltEndIter = _layout->findAbsoluteAtom("__rela_iplt_end");

  auto startEnd = [&](typename DefaultLayout<ELFT>::AbsoluteAtomIterT start,
                      typename DefaultLayout<ELFT>::AbsoluteAtomIterT end,
                      StringRef sec) -> void {
    auto section = _layout->findOutputSection(sec);
    if (section) {
      (*start)->_virtualAddr = section->virtualAddr();
      (*end)->_virtualAddr = section->virtualAddr() + section->memSize();
    } else {
      (*start)->_virtualAddr = 0;
      (*end)->_virtualAddr = 0;
    }
  };

  startEnd(initArrayStartIter, initArrayEndIter, ".init_array");
  startEnd(realIpltStartIter, realIpltEndIter, ".rela.plt");

  assert(!(bssStartAtomIter == _layout->absoluteAtoms().end() ||
           bssEndAtomIter == _layout->absoluteAtoms().end() ||
           underScoreEndAtomIter == _layout->absoluteAtoms().end() ||
           endAtomIter == _layout->absoluteAtoms().end()) &&
         "Unable to find the absolute atoms that have been added by lld");

  auto phe = _programHeader->findProgramHeader(
      llvm::ELF::PT_LOAD, llvm::ELF::PF_W, llvm::ELF::PF_X);

  assert(!(phe == _programHeader->end()) &&
         "Can't find a data segment in the program header!");

  (*bssStartAtomIter)->_virtualAddr = (*phe)->p_vaddr + (*phe)->p_filesz;
  (*bssEndAtomIter)->_virtualAddr = (*phe)->p_vaddr + (*phe)->p_memsz;
  (*underScoreEndAtomIter)->_virtualAddr = (*phe)->p_vaddr + (*phe)->p_memsz;
  (*endAtomIter)->_virtualAddr = (*phe)->p_vaddr + (*phe)->p_memsz;
}

template<class ELFT>
error_code
ExecutableWriter<ELFT>::writeFile(const File &file, StringRef path) {
  buildChunks(file);
  // Create the default sections like the symbol table, string table, and the
  // section string table
  createDefaultSections();

  // Set the Layout
  _layout->assignSectionsToSegments();
  _layout->assignFileOffsets();
  _layout->assignVirtualAddress();

  // Finalize the default value of symbols that the linker adds
  finalizeDefaultAtomValues();

  // Build the Atom To Address map for applying relocations
  buildAtomToAddressMap();

  // Create symbol table and section string table
  buildSymbolTable();

  // add other symbols
  addAbsoluteUndefinedSymbols(file);

  // Finalize the layout by calling the finalize() functions
  _layout->finalize();

  // build Section Header table
  buildSectionHeaderTable();

  // assign Offsets and virtual addresses
  // for sections with no segments
  assignSectionsWithNoSegments();

  uint64_t totalSize = _shdrtab->fileOffset() + _shdrtab->fileSize();

  OwningPtr<FileOutputBuffer> buffer;
  error_code ec = FileOutputBuffer::create(path,
                                           totalSize, buffer,
                                           FileOutputBuffer::F_executable);
  if (ec)
    return ec;

  _Header->e_ident(ELF::EI_CLASS, _targetInfo.is64Bits() ? ELF::ELFCLASS64 :
                       ELF::ELFCLASS32);
  _Header->e_ident(ELF::EI_DATA, _targetInfo.isLittleEndian() ?
                       ELF::ELFDATA2LSB : ELF::ELFDATA2MSB);
  _Header->e_type(_targetInfo.getOutputType());
  _Header->e_machine(_targetInfo.getOutputMachine());

  if (!_targetHandler.doesOverrideHeader()) {
    _Header->e_ident(ELF::EI_VERSION, 1);
    _Header->e_ident(ELF::EI_OSABI, 0);
    _Header->e_version(1);
  } else {
    // override the contents of the ELF Header
    _targetHandler.setHeaderInfo(_Header);
  }
  _Header->e_phoff(_programHeader->fileOffset());
  _Header->e_shoff(_shdrtab->fileOffset());
  _Header->e_phentsize(_programHeader->entsize());
  _Header->e_phnum(_programHeader->numHeaders());
  _Header->e_shentsize(_shdrtab->entsize());
  _Header->e_shnum(_shdrtab->numHeaders());
  _Header->e_shstrndx(_shstrtab->ordinal());
  uint64_t virtualAddr = 0;
  _layout->findAtomAddrByName(_targetInfo.getEntry(), virtualAddr);
  _Header->e_entry(virtualAddr);

  // HACK: We have to write out the header and program header here even though
  // they are a member of a segment because only sections are written in the
  // following loop.
  _Header->write(this, *buffer);
  _programHeader->write(this, *buffer);

  for (auto section : _layout->sections())
    section->write(this, *buffer);

  return buffer->commit();
}

template<class ELFT>
void ExecutableWriter<ELFT>::createDefaultSections() {
  _Header = new Header<ELFT>(_targetInfo);
  _programHeader = new ProgramHeader<ELFT>(_targetInfo);
  _layout->setHeader(_Header);
  _layout->setProgramHeader(_programHeader);

  _symtab = new SymbolTable<
      ELFT>(_targetInfo, ".symtab", DefaultLayout<ELFT>::ORDER_SYMBOL_TABLE);
  _strtab = new StringTable<
      ELFT>(_targetInfo, ".strtab", DefaultLayout<ELFT>::ORDER_STRING_TABLE);
  _shstrtab = new StringTable<ELFT>(
      _targetInfo, ".shstrtab", DefaultLayout<ELFT>::ORDER_SECTION_STRINGS);
  _shdrtab = new SectionHeader<
      ELFT>(_targetInfo, DefaultLayout<ELFT>::ORDER_SECTION_HEADERS);
  _layout->addSection(_symtab);
  _layout->addSection(_strtab);
  _layout->addSection(_shstrtab);
  _shdrtab->setStringSection(_shstrtab);
  _symtab->setStringSection(_strtab);
  _layout->addSection(_shdrtab);

  // give a chance for the target to add sections
  _targetHandler.createDefaultSections();
}
} // namespace elf

std::unique_ptr<Writer> createWriterELF(const ELFTargetInfo &TI) {
  using llvm::object::ELFType;
  // Set the default layout to be the static executable layout
  // We would set the layout to a dynamic executable layout
  // if we came across any shared libraries in the process

  if (!TI.is64Bits() && TI.isLittleEndian())
    return std::unique_ptr<Writer>(new
        elf::ExecutableWriter<ELFType<support::little, 4, false>>(TI));
  else if (TI.is64Bits() && TI.isLittleEndian())
    return std::unique_ptr<Writer>(new
        elf::ExecutableWriter<ELFType<support::little, 8, true>>(TI));
  else if (!TI.is64Bits() && !TI.isLittleEndian())
    return std::unique_ptr<Writer>(new
        elf::ExecutableWriter<ELFType<support::big, 4, false>>(TI));
  else if (TI.is64Bits() && !TI.isLittleEndian())
    return std::unique_ptr<Writer>(new
        elf::ExecutableWriter<ELFType<support::big, 8, true>>(TI));

  llvm_unreachable("Invalid Options!");
}
} // namespace lld
<commit_msg>[ELF] Add more absolute atoms. Simplify how they are resolved.<commit_after>//===- lib/ReaderWriter/ELF/WriterELF.cpp ---------------------------------===//
//
//                             The LLVM Linker
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#include "lld/ReaderWriter/Writer.h"

#include "DefaultLayout.h"
#include "TargetLayout.h"
#include "ExecutableAtoms.h"

#include "lld/ReaderWriter/ELFTargetInfo.h"

using namespace llvm;
using namespace llvm::object;
namespace lld {
namespace elf {
template<class ELFT>
class ExecutableWriter;

//===----------------------------------------------------------------------===//
//  ExecutableWriter Class
//===----------------------------------------------------------------------===//
template<class ELFT>
class ExecutableWriter : public ELFWriter {
public:
  typedef Elf_Shdr_Impl<ELFT> Elf_Shdr;
  typedef Elf_Sym_Impl<ELFT> Elf_Sym;

  ExecutableWriter(const ELFTargetInfo &ti);

private:
  // build the sections that need to be created
  void buildChunks(const File &file);
  virtual error_code writeFile(const File &File, StringRef path);
  void buildAtomToAddressMap();
  void buildSymbolTable ();
  void buildSectionHeaderTable();
  void assignSectionsWithNoSegments();
  void addAbsoluteUndefinedSymbols(const File &File);
  void addDefaultAtoms();
  void addFiles(InputFiles&);
  void finalizeDefaultAtomValues();

  uint64_t addressOfAtom(const Atom *atom) {
    return _atomToAddressMap[atom];
  }

  void createDefaultSections();

  const ELFTargetInfo &_targetInfo;
  TargetHandler<ELFT> &_targetHandler;

  typedef llvm::DenseMap<const Atom *, uint64_t> AtomToAddress;
  AtomToAddress _atomToAddressMap;
  llvm::BumpPtrAllocator _chunkAllocate;
  TargetLayout<ELFT> *_layout;
  Header<ELFT> *_Header;
  ProgramHeader<ELFT> *_programHeader;
  SymbolTable<ELFT> * _symtab;
  StringTable<ELFT> *_strtab;
  StringTable<ELFT> *_shstrtab;
  SectionHeader<ELFT> *_shdrtab;
  CRuntimeFile<ELFT> _runtimeFile;
};

//===----------------------------------------------------------------------===//
//  ExecutableWriter
//===----------------------------------------------------------------------===//
template <class ELFT>
ExecutableWriter<ELFT>::ExecutableWriter(const ELFTargetInfo &ti)
    : _targetInfo(ti), _targetHandler(ti.getTargetHandler<ELFT>()),
      _runtimeFile(ti) {
  _layout = &_targetHandler.targetLayout();
}

template <class ELFT>
void ExecutableWriter<ELFT>::buildChunks(const File &file) {
  for (const DefinedAtom *definedAtom : file.defined() ) {
    _layout->addAtom(definedAtom);
  }
  /// Add all the absolute atoms to the layout
  for (const AbsoluteAtom *absoluteAtom : file.absolute()) {
    _layout->addAtom(absoluteAtom);
  }
}

template<class ELFT>
void ExecutableWriter<ELFT>::buildSymbolTable () {
  for (auto sec : _layout->sections())
    if (auto section = dyn_cast<Section<ELFT>>(sec))
      for (const auto &atom : section->atoms())
        _symtab->addSymbol(atom->_atom, section->ordinal(), atom->_virtualAddr);
}

template<class ELFT>
void
ExecutableWriter<ELFT>::addAbsoluteUndefinedSymbols(const File &file) {
  // add all the absolute symbols that the layout contains to the output symbol
  // table
  for (auto &atom : _layout->absoluteAtoms())
    _symtab->addSymbol(atom->_atom, ELF::SHN_ABS, atom->_virtualAddr);
  for (const UndefinedAtom *a : file.undefined())
    _symtab->addSymbol(a, ELF::SHN_UNDEF);
}

template<class ELFT>
void ExecutableWriter<ELFT>::buildAtomToAddressMap () {
  for (auto sec : _layout->sections())
    if (auto section = dyn_cast<Section<ELFT>>(sec))
      for (const auto &atom : section->atoms())
        _atomToAddressMap[atom->_atom] = atom->_virtualAddr;
  // build the atomToAddressMap that contains absolute symbols too
  for (auto &atom : _layout->absoluteAtoms())
    _atomToAddressMap[atom->_atom] = atom->_virtualAddr;
}

template<class ELFT>
void ExecutableWriter<ELFT>::buildSectionHeaderTable() {
  for (auto mergedSec : _layout->mergedSections()) {
    if (mergedSec->kind() != Chunk<ELFT>::K_ELFSection)
      continue;
    if (mergedSec->hasSegment())
      _shdrtab->appendSection(mergedSec);
  }
}

template<class ELFT>
void ExecutableWriter<ELFT>::assignSectionsWithNoSegments() {
  for (auto mergedSec : _layout->mergedSections()) {
    if (mergedSec->kind() != Chunk<ELFT>::K_ELFSection)
      continue;
    if (!mergedSec->hasSegment())
      _shdrtab->appendSection(mergedSec);
  }
  _layout->assignOffsetsForMiscSections();
  for (auto sec : _layout->sections())
    if (auto section = dyn_cast<Section<ELFT>>(sec))
      if (!DefaultLayout<ELFT>::hasOutputSegment(section))
        _shdrtab->updateSection(section);
}

/// \brief Add absolute symbols by default. These are linker added
/// absolute symbols
template<class ELFT>
void ExecutableWriter<ELFT>::addDefaultAtoms() {
  _runtimeFile.addUndefinedAtom(_targetInfo.getEntry());
  _runtimeFile.addAbsoluteAtom("__bss_start");
  _runtimeFile.addAbsoluteAtom("__bss_end");
  _runtimeFile.addAbsoluteAtom("_end");
  _runtimeFile.addAbsoluteAtom("end");
  _runtimeFile.addAbsoluteAtom("__preinit_array_start");
  _runtimeFile.addAbsoluteAtom("__preinit_array_end");
  _runtimeFile.addAbsoluteAtom("__init_array_start");
  _runtimeFile.addAbsoluteAtom("__init_array_end");
  _runtimeFile.addAbsoluteAtom("__rela_iplt_start");
  _runtimeFile.addAbsoluteAtom("__rela_iplt_end");
  _runtimeFile.addAbsoluteAtom("__fini_array_start");
  _runtimeFile.addAbsoluteAtom("__fini_array_end");
}

/// \brief Hook in lld to add CRuntime file
template <class ELFT>
void ExecutableWriter<ELFT>::addFiles(InputFiles &inputFiles) {
  addDefaultAtoms();
  inputFiles.prependFile(_runtimeFile);
  // Give a chance for the target to add atoms
  _targetHandler.addFiles(inputFiles);
}

/// Finalize the value of all the absolute symbols that we
/// created
template<class ELFT>
void ExecutableWriter<ELFT>::finalizeDefaultAtomValues() {
  auto bssStartAtomIter = _layout->findAbsoluteAtom("__bss_start");
  auto bssEndAtomIter = _layout->findAbsoluteAtom("__bss_end");
  auto underScoreEndAtomIter = _layout->findAbsoluteAtom("_end");
  auto endAtomIter = _layout->findAbsoluteAtom("end");

  auto startEnd = [&](StringRef sym, StringRef sec) -> void {
    // TODO: This looks like a good place to use Twine...
    std::string start("__"), end("__");
    start += sym;
    start += "_start";
    end += sym;
    end += "_end";
    auto s = _layout->findAbsoluteAtom(start);
    auto e = _layout->findAbsoluteAtom(end);
    auto section = _layout->findOutputSection(sec);
    if (section) {
      (*s)->_virtualAddr = section->virtualAddr();
      (*e)->_virtualAddr = section->virtualAddr() + section->memSize();
    } else {
      (*s)->_virtualAddr = 0;
      (*e)->_virtualAddr = 0;
    }
  };

  startEnd("preinit_array", ".preinit_array");
  startEnd("init_array", ".init_array");
  startEnd("rela_iplt", ".rela.plt");
  startEnd("fini_array", ".fini_array");

  assert(!(bssStartAtomIter == _layout->absoluteAtoms().end() ||
           bssEndAtomIter == _layout->absoluteAtoms().end() ||
           underScoreEndAtomIter == _layout->absoluteAtoms().end() ||
           endAtomIter == _layout->absoluteAtoms().end()) &&
         "Unable to find the absolute atoms that have been added by lld");

  auto phe = _programHeader->findProgramHeader(
      llvm::ELF::PT_LOAD, llvm::ELF::PF_W, llvm::ELF::PF_X);

  assert(!(phe == _programHeader->end()) &&
         "Can't find a data segment in the program header!");

  (*bssStartAtomIter)->_virtualAddr = (*phe)->p_vaddr + (*phe)->p_filesz;
  (*bssEndAtomIter)->_virtualAddr = (*phe)->p_vaddr + (*phe)->p_memsz;
  (*underScoreEndAtomIter)->_virtualAddr = (*phe)->p_vaddr + (*phe)->p_memsz;
  (*endAtomIter)->_virtualAddr = (*phe)->p_vaddr + (*phe)->p_memsz;
}

template<class ELFT>
error_code
ExecutableWriter<ELFT>::writeFile(const File &file, StringRef path) {
  buildChunks(file);
  // Create the default sections like the symbol table, string table, and the
  // section string table
  createDefaultSections();

  // Set the Layout
  _layout->assignSectionsToSegments();
  _layout->assignFileOffsets();
  _layout->assignVirtualAddress();

  // Finalize the default value of symbols that the linker adds
  finalizeDefaultAtomValues();

  // Build the Atom To Address map for applying relocations
  buildAtomToAddressMap();

  // Create symbol table and section string table
  buildSymbolTable();

  // add other symbols
  addAbsoluteUndefinedSymbols(file);

  // Finalize the layout by calling the finalize() functions
  _layout->finalize();

  // build Section Header table
  buildSectionHeaderTable();

  // assign Offsets and virtual addresses
  // for sections with no segments
  assignSectionsWithNoSegments();

  uint64_t totalSize = _shdrtab->fileOffset() + _shdrtab->fileSize();

  OwningPtr<FileOutputBuffer> buffer;
  error_code ec = FileOutputBuffer::create(path,
                                           totalSize, buffer,
                                           FileOutputBuffer::F_executable);
  if (ec)
    return ec;

  _Header->e_ident(ELF::EI_CLASS, _targetInfo.is64Bits() ? ELF::ELFCLASS64 :
                       ELF::ELFCLASS32);
  _Header->e_ident(ELF::EI_DATA, _targetInfo.isLittleEndian() ?
                       ELF::ELFDATA2LSB : ELF::ELFDATA2MSB);
  _Header->e_type(_targetInfo.getOutputType());
  _Header->e_machine(_targetInfo.getOutputMachine());

  if (!_targetHandler.doesOverrideHeader()) {
    _Header->e_ident(ELF::EI_VERSION, 1);
    _Header->e_ident(ELF::EI_OSABI, 0);
    _Header->e_version(1);
  } else {
    // override the contents of the ELF Header
    _targetHandler.setHeaderInfo(_Header);
  }
  _Header->e_phoff(_programHeader->fileOffset());
  _Header->e_shoff(_shdrtab->fileOffset());
  _Header->e_phentsize(_programHeader->entsize());
  _Header->e_phnum(_programHeader->numHeaders());
  _Header->e_shentsize(_shdrtab->entsize());
  _Header->e_shnum(_shdrtab->numHeaders());
  _Header->e_shstrndx(_shstrtab->ordinal());
  uint64_t virtualAddr = 0;
  _layout->findAtomAddrByName(_targetInfo.getEntry(), virtualAddr);
  _Header->e_entry(virtualAddr);

  // HACK: We have to write out the header and program header here even though
  // they are a member of a segment because only sections are written in the
  // following loop.
  _Header->write(this, *buffer);
  _programHeader->write(this, *buffer);

  for (auto section : _layout->sections())
    section->write(this, *buffer);

  return buffer->commit();
}

template<class ELFT>
void ExecutableWriter<ELFT>::createDefaultSections() {
  _Header = new Header<ELFT>(_targetInfo);
  _programHeader = new ProgramHeader<ELFT>(_targetInfo);
  _layout->setHeader(_Header);
  _layout->setProgramHeader(_programHeader);

  _symtab = new SymbolTable<
      ELFT>(_targetInfo, ".symtab", DefaultLayout<ELFT>::ORDER_SYMBOL_TABLE);
  _strtab = new StringTable<
      ELFT>(_targetInfo, ".strtab", DefaultLayout<ELFT>::ORDER_STRING_TABLE);
  _shstrtab = new StringTable<ELFT>(
      _targetInfo, ".shstrtab", DefaultLayout<ELFT>::ORDER_SECTION_STRINGS);
  _shdrtab = new SectionHeader<
      ELFT>(_targetInfo, DefaultLayout<ELFT>::ORDER_SECTION_HEADERS);
  _layout->addSection(_symtab);
  _layout->addSection(_strtab);
  _layout->addSection(_shstrtab);
  _shdrtab->setStringSection(_shstrtab);
  _symtab->setStringSection(_strtab);
  _layout->addSection(_shdrtab);

  // give a chance for the target to add sections
  _targetHandler.createDefaultSections();
}
} // namespace elf

std::unique_ptr<Writer> createWriterELF(const ELFTargetInfo &TI) {
  using llvm::object::ELFType;
  // Set the default layout to be the static executable layout
  // We would set the layout to a dynamic executable layout
  // if we came across any shared libraries in the process

  if (!TI.is64Bits() && TI.isLittleEndian())
    return std::unique_ptr<Writer>(new
        elf::ExecutableWriter<ELFType<support::little, 4, false>>(TI));
  else if (TI.is64Bits() && TI.isLittleEndian())
    return std::unique_ptr<Writer>(new
        elf::ExecutableWriter<ELFType<support::little, 8, true>>(TI));
  else if (!TI.is64Bits() && !TI.isLittleEndian())
    return std::unique_ptr<Writer>(new
        elf::ExecutableWriter<ELFType<support::big, 4, false>>(TI));
  else if (TI.is64Bits() && !TI.isLittleEndian())
    return std::unique_ptr<Writer>(new
        elf::ExecutableWriter<ELFType<support::big, 8, true>>(TI));

  llvm_unreachable("Invalid Options!");
}
} // namespace lld
<|endoftext|>