<commit_before>/*
 * Copyright 2013-2016 Christian Lockley
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License. You may obtain
 * a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */
#define _DEFAULT_SOURCE
#include "watchdogd.hpp"
#include "sub.hpp"
#include <dirent.h>
#include <semaphore.h>
#include "testdir.hpp"
#include "exe.hpp"
#include "repair.hpp"
#include <sys/eventfd.h>
#include "threadpool.hpp"
#include "futex.hpp"

static std::atomic_int sem = {0};
//The dirent_buf_size function was written by Ben Hutchings and released under the following license.

//Permission is hereby granted, free of charge, to any person obtaining a copy of this
//software and associated documentation files (the "Software"), to deal in the Software
//without restriction, including without limitation the rights to use, copy, modify, merge,
//publish, distribute, sublicense, and/or sell copies of the Software, and to permit
//persons to whom the Software is furnished to do so, subject to the following
//condition:
//THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
//EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
//MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
//NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
//HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
//WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
//DEALINGS IN THE SOFTWARE.

//Copied into this program Sun September 8, 2013 by Christian Lockley

size_t DirentBufSize(DIR * dirp)
{
	long name_max;
	size_t name_end;
#if defined(HAVE_FPATHCONF) && defined(HAVE_DIRFD) \
&& defined(_PC_NAME_MAX)
	name_max = fpathconf(dirfd(dirp), _PC_NAME_MAX);
	if (name_max == -1)
#if defined(NAME_MAX)
		name_max = (NAME_MAX > 255) ? NAME_MAX : 255;
#else
		return (size_t) (-1);
#endif
#else
#if defined(NAME_MAX)
	name_max = (NAME_MAX > 255) ? NAME_MAX : 255;
#else
#error "buffer size for readdir_r cannot be determined"
#endif
#endif
	name_end =
	    (size_t) offsetof(struct dirent,
			      d_name) + (unsigned long)name_max + 1;
	return (name_end > sizeof(struct dirent)
		? name_end : sizeof(struct dirent));
}

static void DeleteDuplicates(ProcessList * p)
{
	assert(p != NULL);

	repaircmd_t *c = NULL;
	repaircmd_t *next = NULL;

	list_for_each_entry(c, next, &p->head, entry) {
		repaircmd_t *b = NULL;
		repaircmd_t *next2 = NULL;
		if (c->legacy) {
			list_for_each_entry(b, next2, &p->head, entry) {
				if (!b->legacy) {
					if (strcmp(c->path, b->path) == 0) {
						list_del(&c->entry);
						free((void *)c->path);
						Logmsg(LOG_INFO, "Using configuration info for %s script", b->path);
					}
				}
			}
		}
	}
}

int CreateLinkedListOfExes(char *repairScriptFolder, ProcessList * p,
			   struct cfgoptions *const config)
{
	assert(p != NULL);
	assert(repairScriptFolder != NULL);

	struct stat buffer;
	struct dirent *ent = NULL;

	list_init(&p->head);

	int fd = open(repairScriptFolder, O_DIRECTORY | O_RDONLY);

	if (fd == -1) {
		if (!(IDENTIFY & config->options)) {
			fprintf(stderr, "watchdogd: %s: %s\n", repairScriptFolder,
				MyStrerror(errno));
		}

		if (config->options & FORCE || config->options & SOFTBOOT || config->haveConfigFile == false) {
			return 0;
		}

		return 1;
	}

	DIR *dir = fdopendir(fd);

	if (dir == NULL) {
		Logmsg(LOG_ERR, "watchdogd: %s", MyStrerror(errno));
		close(fd);
		return -1;
	}

	errno = 0;

	while ((ent = readdir(dir)) != NULL) {

		int statfd = dirfd(dir);

		if (statfd == -1) {
			goto error;
		}

		if (strchr(".", ent->d_name[0]) != NULL && !(config->options & FORCE)) {
			continue;
		}

		if (fstatat(statfd, ent->d_name, &buffer, 0) < 0)
			continue;

		if (IsRepairScriptConfig(ent->d_name) == 0) {
			if (S_ISREG(buffer.st_mode) == 0) {
				continue;
			}

			if (!(buffer.st_mode & S_IXUSR))
				continue;

			if (!(buffer.st_mode & S_IRUSR))
				continue;
		} else {
			if (S_ISREG(buffer.st_mode) == 0) {
				continue;
			}
		}

		repaircmd_t *cmd = (repaircmd_t *) calloc(1, sizeof(*cmd));

		if (cmd == NULL) {
			goto error;
		}

		for (size_t i = 0; i < strlen(repairScriptFolder); i += 1) {
			if (repairScriptFolder[i] == '/' && i < strlen(repairScriptFolder)
			    && repairScriptFolder[i+1] == '\0') {
				repairScriptFolder[i] = '\0';
			}
		}

		Wasprintf((char **)&cmd->path, "%s/%s", repairScriptFolder, ent->d_name);

		if (cmd->path == NULL) {
			assert(cmd != NULL);
			free(cmd);
			goto error;
		}

		if (IsRepairScriptConfig(ent->d_name) == 0) {
			cmd->legacy = true;
		} else {
			//For V3 repair scripts cmd->path refers to the pathname of the repair script config file
			if (LoadRepairScriptLink
			    (&cmd->spawnattr, (char *)cmd->path) == false) {
				free((void *)cmd->path);
				free(cmd);
				continue;
			}

			cmd->spawnattr.repairFilePathname = strdup(cmd->path);
			free((void *)cmd->path);
			cmd->path = NULL;

			cmd->path = cmd->spawnattr.execStart;

			if (cmd->path == NULL) {
				fprintf(stderr,
					"Ignoring malformed repair file: %s\n",
					ent->d_name);
				free(cmd);
				continue;
			}

			if (fd < 0) {
				fprintf(stderr, "unable to open file %s:\n",
					MyStrerror(errno));
				free((void *)cmd->path);
				free((void *)cmd);
				continue;
			}

			if (IsExe(cmd->path, false) < 0) {
				fprintf(stderr, "%s is not an executable\n",
					cmd->path);
				free((void *)cmd->path);
				free((void *)cmd);
				continue;
			}

			cmd->legacy = false;
		}

		list_add(&cmd->entry, &p->head);
	}

	assert(fd != -1);

	close(fd);
	closedir(dir);
	DeleteDuplicates(p);
	return 0;

 error:
	assert(fd != -1);
	close(fd);
	Logmsg(LOG_ERR, "watchdogd: %s", MyStrerror(errno));
	closedir(dir);

	return -1;
}

void FreeExeList(ProcessList * p)
{
	assert(p != NULL);

	repaircmd_t *c = NULL;
	repaircmd_t *next = NULL;

	list_for_each_entry(c, next, &p->head, entry) {
		list_del(&c->entry);
		free((void *)c->path);
		if (c->legacy == false) {
			free((void *)c->spawnattr.user);
			free((void *)c->spawnattr.group);
			free((void *)c->spawnattr.workingDirectory);
			free((void *)c->spawnattr.repairFilePathname);
			free((void *)c->spawnattr.noNewPrivileges);
		}
		free(c);
	}
}

static void * __ExecScriptWorkerThread(void *a)
{
	assert(a != NULL);

	__sync_synchronize();

	Container *container = (Container *) a;
	repaircmd_t *c = container->cmd;
	container->workerThreadCount += 1;
	sem = 1;
	FutexWake(&sem);
	__sync_synchronize();

	if (c->legacy == false) {
		if (c->retString[0] == '\0') {
			c->ret =
			    SpawnAttr(&c->spawnattr, c->path, c->path, c->mode == TEST ? "test": "repair",
				      NULL);
		} else {
			c->ret =
			    SpawnAttr(&c->spawnattr, c->path, c->path, c->mode == TEST ? "test": "repair", c->retString,
				      NULL);
		}
	} else {
		spawnattr_t attr = {
				.workingDirectory = NULL, .repairFilePathname = NULL,
	        		.execStart = NULL, .user = NULL, .group = NULL,
				.timeout = container->config->repairBinTimeout, .nice = 0, .umask = 0,
				.noNewPrivileges = false, .hasUmask = false
		};

		if (c->retString[0] == '\0') {
			if (c->mode == TEST) {
				c->ret = SpawnAttr(&attr, c->path, c->path, "test", NULL);
			} else {
				c->ret = SpawnAttr(&attr, c->path, c->path, "repair", c->path, NULL);
			}
		} else {
			if (c->mode == TEST) {
				c->ret = SpawnAttr(&attr, c->path, c->path, "test", NULL);
			} else {
				c->ret = SpawnAttr(&attr, c->path, c->path, "repair", c->retString, c->path, NULL);
			}
		}
	}

	c->retString[0] = '\0';

	container->workerThreadCount -= 1;

	__sync_synchronize();
	return NULL;
}

static void __WaitForWorkers(Container const *container)
{
	while (container->workerThreadCount != 0) {
		sleep(1);
	}
}

static int __ExecuteRepairScripts(void *a)
{
	struct executeScriptsStruct *arg = (struct executeScriptsStruct *)a;
	ProcessList *p = arg->list;
	struct cfgoptions *s = arg->config;

	Container container = {{0}, s, NULL};

	repaircmd_t *c = NULL;
	repaircmd_t *next = NULL;

	list_for_each_entry(c, next, &p->head, entry) {
		container.cmd = c;
		container.cmd->mode = TEST;

		c->retString[0] = '\0';
		ThreadPoolAddTask(__ExecScriptWorkerThread, &container, true);
		__sync_synchronize();
		FutexWait(&sem, 0);
		sem = 0;
	}

	c = NULL;
	next = NULL;

	__WaitForWorkers(&container);

	list_for_each_entry(c, next, &p->head, entry) {
		if (c->ret == 0) {
			continue;
		}

		container.cmd = c;
		portable_snprintf(container.cmd->retString, sizeof(container.cmd->retString), "%i", c->ret);

		container.cmd->mode = REPAIR;

		ThreadPoolAddTask(__ExecScriptWorkerThread, &container, true);
		__sync_synchronize();
		FutexWait(&sem, 0);
		sem = 0;
	}

	__WaitForWorkers(&container);

	c = NULL;
	next = NULL;

	list_for_each_entry(c, next, &p->head, entry) {
		if (c->ret != 0) {
			Logmsg(LOG_ERR, "repair %s script failed",
				c->legacy == false ? c->spawnattr.repairFilePathname : c->path);
			return 1;
		}
	}

	return 0;
}

static int pipeFd[2] = {0};

bool ExecuteRepairScriptsPreFork(ProcessList * p, struct cfgoptions *s)
{
	if (list_is_empty(&p->head)) {
		return true;
	}

	pipe(pipeFd);

	pid_t pid = fork();

	if (pid == -1) {
		Logmsg(LOG_ERR, "%s\n", MyStrerror(errno));
		return false;
	} else if (pid == 0) {
		close(pipeFd[0]);
		pid_t pid = fork();

#if defined(__linux__)
		if (LinuxRunningSystemd() == 1) {
			unsetenv("NOTIFY_SOCKET");
		}
#endif

		unsetenv("LD_PRELOAD");

		if (pid > 0) {
			_Exit(0);
		}

		if (pid < 0) {
			Logmsg(LOG_ERR, "fork failed: %s", MyStrerror(errno));
			abort();
		}

		char b[1];

		ThreadPoolNew();

		while (true) {
			uint64_t value = 0;
			struct executeScriptsStruct ess;
			ess.list = p;
			ess.config = s;

			int ret = __ExecuteRepairScripts(&ess);

			write(pipeFd[1], &ret, sizeof(ret));
		}

		exit(0);
	}
	close(pipeFd[1]);
	return true;
}

int ExecuteRepairScripts(void)
{
	uint64_t value = 1;
	int ret = 0;

	read(pipeFd[0], &ret, sizeof(ret));

	if (ret != 0) {
		return -1;
	}

	return 0;
}
<commit_msg>allow relative paths in .repair file<commit_after>/*
 * Copyright 2013-2016 Christian Lockley
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License. You may obtain
 * a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */
#define _DEFAULT_SOURCE
#include <libgen.h>
#include "watchdogd.hpp"
#include "sub.hpp"
#include <dirent.h>
#include <semaphore.h>
#include "testdir.hpp"
#include "exe.hpp"
#include "repair.hpp"
#include <sys/eventfd.h>
#include "threadpool.hpp"
#include "futex.hpp"

static std::atomic_int sem = {0};
//The dirent_buf_size function was written by Ben Hutchings and released under the following license.

//Permission is hereby granted, free of charge, to any person obtaining a copy of this
//software and associated documentation files (the "Software"), to deal in the Software
//without restriction, including without limitation the rights to use, copy, modify, merge,
//publish, distribute, sublicense, and/or sell copies of the Software, and to permit
//persons to whom the Software is furnished to do so, subject to the following
//condition:
//THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
//EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
//MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
//NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
//HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
//WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
//DEALINGS IN THE SOFTWARE.

//Copied into this program Sun September 8, 2013 by Christian Lockley

size_t DirentBufSize(DIR * dirp)
{
	long name_max;
	size_t name_end;
#if defined(HAVE_FPATHCONF) && defined(HAVE_DIRFD) \
&& defined(_PC_NAME_MAX)
	name_max = fpathconf(dirfd(dirp), _PC_NAME_MAX);
	if (name_max == -1)
#if defined(NAME_MAX)
		name_max = (NAME_MAX > 255) ? NAME_MAX : 255;
#else
		return (size_t) (-1);
#endif
#else
#if defined(NAME_MAX)
	name_max = (NAME_MAX > 255) ? NAME_MAX : 255;
#else
#error "buffer size for readdir_r cannot be determined"
#endif
#endif
	name_end =
	    (size_t) offsetof(struct dirent,
			      d_name) + (unsigned long)name_max + 1;
	return (name_end > sizeof(struct dirent)
		? name_end : sizeof(struct dirent));
}

static void DeleteDuplicates(ProcessList * p)
{
	assert(p != NULL);

	repaircmd_t *c = NULL;
	repaircmd_t *next = NULL;

	list_for_each_entry(c, next, &p->head, entry) {
		repaircmd_t *b = NULL;
		repaircmd_t *next2 = NULL;
		if (c->legacy) {
			list_for_each_entry(b, next2, &p->head, entry) {
				if (!b->legacy) {
					if (strcmp(c->path, b->path) == 0) {
						list_del(&c->entry);
						free((void *)c->path);
						Logmsg(LOG_INFO, "Using configuration info for %s script", b->path);
					}
				}
			}
		}
	}
}

int CreateLinkedListOfExes(char *repairScriptFolder, ProcessList * p,
			   struct cfgoptions *const config)
{
	assert(p != NULL);
	assert(repairScriptFolder != NULL);

	struct stat buffer;
	struct dirent *ent = NULL;

	list_init(&p->head);

	int fd = open(repairScriptFolder, O_DIRECTORY | O_RDONLY);

	if (fd == -1) {
		if (!(IDENTIFY & config->options)) {
			fprintf(stderr, "watchdogd: %s: %s\n", repairScriptFolder,
				MyStrerror(errno));
		}

		if (config->options & FORCE || config->options & SOFTBOOT || config->haveConfigFile == false) {
			return 0;
		}

		return 1;
	}

	DIR *dir = fdopendir(fd);

	if (dir == NULL) {
		Logmsg(LOG_ERR, "watchdogd: %s", MyStrerror(errno));
		close(fd);
		return -1;
	}

	errno = 0;

	while ((ent = readdir(dir)) != NULL) {

		int statfd = dirfd(dir);

		if (statfd == -1) {
			goto error;
		}

		if (strchr(".", ent->d_name[0]) != NULL && !(config->options & FORCE)) {
			continue;
		}

		if (fstatat(statfd, ent->d_name, &buffer, 0) < 0)
			continue;

		if (IsRepairScriptConfig(ent->d_name) == 0) {
			if (S_ISREG(buffer.st_mode) == 0) {
				continue;
			}

			if (!(buffer.st_mode & S_IXUSR))
				continue;

			if (!(buffer.st_mode & S_IRUSR))
				continue;
		} else {
			if (S_ISREG(buffer.st_mode) == 0) {
				continue;
			}
		}

		repaircmd_t *cmd = (repaircmd_t *) calloc(1, sizeof(*cmd));

		if (cmd == NULL) {
			goto error;
		}

		for (size_t i = 0; i < strlen(repairScriptFolder); i += 1) {
			if (repairScriptFolder[i] == '/' && i < strlen(repairScriptFolder)
			    && repairScriptFolder[i+1] == '\0') {
				repairScriptFolder[i] = '\0';
			}
		}

		Wasprintf((char **)&cmd->path, "%s/%s", repairScriptFolder, ent->d_name);

		if (cmd->path == NULL) {
			assert(cmd != NULL);
			free(cmd);
			goto error;
		}

		if (IsRepairScriptConfig(ent->d_name) == 0) {
			cmd->legacy = true;
		} else {
			//For V3 repair scripts cmd->path refers to the pathname of the repair script config file
			if (LoadRepairScriptLink
			    (&cmd->spawnattr, (char *)cmd->path) == false) {
				free((void *)cmd->path);
				free(cmd);
				continue;
			}

			cmd->spawnattr.repairFilePathname = strdup(cmd->path);
			free((void *)cmd->path);
			cmd->path = NULL;
			cmd->path = cmd->spawnattr.execStart;

			char * rel = basename(strdup(cmd->path));

			if (strcmp(rel, cmd->path) == 0) {
				free((void*)rel);
				Wasprintf((char **)&rel, "%s/%s", repairScriptFolder, cmd->path);
				free((void*)cmd->path);
				cmd->path = cmd->spawnattr.execStart = rel;
			}

			if (cmd->path == NULL) {
				fprintf(stderr,
					"Ignoring malformed repair file: %s\n",
					ent->d_name);
				free(cmd);
				continue;
			}

			if (fd < 0) {
				fprintf(stderr, "unable to open file %s:\n",
					MyStrerror(errno));
				free((void *)cmd->path);
				free((void *)cmd);
				continue;
			}

			if (IsExe(cmd->path, false) < 0) {
				fprintf(stderr, "%s is not an executable\n",
					cmd->path);
				free((void *)cmd->path);
				free((void *)cmd);
				continue;
			}

			cmd->legacy = false;
		}

		list_add(&cmd->entry, &p->head);
	}

	assert(fd != -1);

	close(fd);
	closedir(dir);
	DeleteDuplicates(p);
	return 0;

 error:
	assert(fd != -1);
	close(fd);
	Logmsg(LOG_ERR, "watchdogd: %s", MyStrerror(errno));
	closedir(dir);

	return -1;
}

void FreeExeList(ProcessList * p)
{
	assert(p != NULL);

	repaircmd_t *c = NULL;
	repaircmd_t *next = NULL;

	list_for_each_entry(c, next, &p->head, entry) {
		list_del(&c->entry);
		free((void *)c->path);
		if (c->legacy == false) {
			free((void *)c->spawnattr.user);
			free((void *)c->spawnattr.group);
			free((void *)c->spawnattr.workingDirectory);
			free((void *)c->spawnattr.repairFilePathname);
			free((void *)c->spawnattr.noNewPrivileges);
		}
		free(c);
	}
}

static void * __ExecScriptWorkerThread(void *a)
{
	assert(a != NULL);

	__sync_synchronize();

	Container *container = (Container *) a;
	repaircmd_t *c = container->cmd;
	container->workerThreadCount += 1;
	sem = 1;
	FutexWake(&sem);
	__sync_synchronize();

	if (c->legacy == false) {
		if (c->retString[0] == '\0') {
			c->ret =
			    SpawnAttr(&c->spawnattr, c->path, c->path, c->mode == TEST ? "test": "repair",
				      NULL);
		} else {
			c->ret =
			    SpawnAttr(&c->spawnattr, c->path, c->path, c->mode == TEST ? "test": "repair", c->retString,
				      NULL);
		}
	} else {
		spawnattr_t attr = {
				.workingDirectory = NULL, .repairFilePathname = NULL,
	        		.execStart = NULL, .user = NULL, .group = NULL,
				.timeout = container->config->repairBinTimeout, .nice = 0, .umask = 0,
				.noNewPrivileges = false, .hasUmask = false
		};

		if (c->retString[0] == '\0') {
			if (c->mode == TEST) {
				c->ret = SpawnAttr(&attr, c->path, c->path, "test", NULL);
			} else {
				c->ret = SpawnAttr(&attr, c->path, c->path, "repair", c->path, NULL);
			}
		} else {
			if (c->mode == TEST) {
				c->ret = SpawnAttr(&attr, c->path, c->path, "test", NULL);
			} else {
				c->ret = SpawnAttr(&attr, c->path, c->path, "repair", c->retString, c->path, NULL);
			}
		}
	}

	c->retString[0] = '\0';

	container->workerThreadCount -= 1;

	__sync_synchronize();
	return NULL;
}

static void __WaitForWorkers(Container const *container)
{
	while (container->workerThreadCount != 0) {
		sleep(1);
	}
}

static int __ExecuteRepairScripts(void *a)
{
	struct executeScriptsStruct *arg = (struct executeScriptsStruct *)a;
	ProcessList *p = arg->list;
	struct cfgoptions *s = arg->config;

	Container container = {{0}, s, NULL};

	repaircmd_t *c = NULL;
	repaircmd_t *next = NULL;

	list_for_each_entry(c, next, &p->head, entry) {
		container.cmd = c;
		container.cmd->mode = TEST;

		c->retString[0] = '\0';
		ThreadPoolAddTask(__ExecScriptWorkerThread, &container, true);
		__sync_synchronize();
		FutexWait(&sem, 0);
		sem = 0;
	}

	c = NULL;
	next = NULL;

	__WaitForWorkers(&container);

	list_for_each_entry(c, next, &p->head, entry) {
		if (c->ret == 0) {
			continue;
		}

		container.cmd = c;
		portable_snprintf(container.cmd->retString, sizeof(container.cmd->retString), "%i", c->ret);

		container.cmd->mode = REPAIR;

		ThreadPoolAddTask(__ExecScriptWorkerThread, &container, true);
		__sync_synchronize();
		FutexWait(&sem, 0);
		sem = 0;
	}

	__WaitForWorkers(&container);

	c = NULL;
	next = NULL;

	list_for_each_entry(c, next, &p->head, entry) {
		if (c->ret != 0) {
			Logmsg(LOG_ERR, "repair %s script failed",
				c->legacy == false ? c->spawnattr.repairFilePathname : c->path);
			return 1;
		}
	}

	return 0;
}

static int pipeFd[2] = {0};

bool ExecuteRepairScriptsPreFork(ProcessList * p, struct cfgoptions *s)
{
	if (list_is_empty(&p->head)) {
		return true;
	}

	pipe(pipeFd);

	pid_t pid = fork();

	if (pid == -1) {
		Logmsg(LOG_ERR, "%s\n", MyStrerror(errno));
		return false;
	} else if (pid == 0) {
		close(pipeFd[0]);
		pid_t pid = fork();

#if defined(__linux__)
		if (LinuxRunningSystemd() == 1) {
			unsetenv("NOTIFY_SOCKET");
		}
#endif

		unsetenv("LD_PRELOAD");

		if (pid > 0) {
			_Exit(0);
		}

		if (pid < 0) {
			Logmsg(LOG_ERR, "fork failed: %s", MyStrerror(errno));
			abort();
		}

		char b[1];

		ThreadPoolNew();

		while (true) {
			uint64_t value = 0;
			struct executeScriptsStruct ess;
			ess.list = p;
			ess.config = s;

			int ret = __ExecuteRepairScripts(&ess);

			write(pipeFd[1], &ret, sizeof(ret));
		}

		exit(0);
	}
	close(pipeFd[1]);
	return true;
}

int ExecuteRepairScripts(void)
{
	uint64_t value = 1;
	int ret = 0;

	read(pipeFd[0], &ret, sizeof(ret));

	if (ret != 0) {
		return -1;
	}

	return 0;
}
<|endoftext|>


<commit_before>/** \brief Handles crawling as well as RSS feeds.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2018 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <iostream>
#include <memory>
#include <unordered_map>
#include <cstdlib>
#include "DbConnection.h"
#include "IniFile.h"
#include "MARC.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "util.h"
#include "Zotero.h"

namespace {

[[noreturn]] void Usage() {
    std::cerr << "Usage: " << ::progname << " [options] config_file_path [section1 section2 .. sectionN]\n"
              << "\n"
              << "\tOptions:\n"
              << "\t[--verbosity=log_level]                                     Possible log levels are ERROR, WARNING, INFO, and DEBUG with the default being WARNING.\n"
              << "\t[--test]                                                    No download information will be stored for further downloads.\n"
              << "\t[--ignore-robots-dot-txt]\n"
              << "\t[--map-directory=map_directory]\n"
              << "\t[--previous-downloads-db-file=previous_downloads_db_file]\n"
              << "\t[--output-file=output_file]\n"
              << "\n"
              << "\tIf any section names have been provided, only those will be processed o/w all sections will be processed.\n\n";
    std::exit(EXIT_FAILURE);
}

void LoadMARCEditInstructions(const IniFile::Section &section, std::vector<MARC::EditInstruction> * edit_instructions) {
    edit_instructions->clear();

    for (const auto &name_and_value : section) {
        if (StringUtil::StartsWith(name_and_value.first, "insert_field_")) {
            const std::string tag_candidate(name_and_value.first.substr(__builtin_strlen("insert_field_")));
            if (tag_candidate.length() != MARC::Record::TAG_LENGTH)
                LOG_ERROR("bad entry in section \"" + section.getSectionName() + "\" \"" + name_and_value.first + "\"!");
            edit_instructions->emplace_back(MARC::EditInstruction::CreateInsertFieldInstruction(tag_candidate, name_and_value.second));
        } else if (StringUtil::StartsWith(name_and_value.first, "insert_subfield_")) {
            const std::string tag_and_subfield_code_candidate(name_and_value.first.substr(__builtin_strlen("insert_subfield_")));
            if (tag_and_subfield_code_candidate.length() != MARC::Record::TAG_LENGTH + 1)
                LOG_ERROR("bad entry in section \"" + section.getSectionName() + "\" \"" + name_and_value.first + "\"!");
            edit_instructions->emplace_back(MARC::EditInstruction::CreateInsertSubfieldInstruction(
                tag_and_subfield_code_candidate.substr(0, MARC::Record::TAG_LENGTH),
                tag_and_subfield_code_candidate[MARC::Record::TAG_LENGTH], name_and_value.second));
        } else if (StringUtil::StartsWith(name_and_value.first, "add_subfield_")) {
            const std::string tag_and_subfield_code_candidate(name_and_value.first.substr(__builtin_strlen("add_subfield_")));
            if (tag_and_subfield_code_candidate.length() != MARC::Record::TAG_LENGTH + 1)
                LOG_ERROR("bad entry in section \"" + section.getSectionName() + "\" \"" + name_and_value.first + "\"!");
            edit_instructions->emplace_back(MARC::EditInstruction::CreateAddSubfieldInstruction(
                tag_and_subfield_code_candidate.substr(0, MARC::Record::TAG_LENGTH),
                tag_and_subfield_code_candidate[MARC::Record::TAG_LENGTH], name_and_value.second));
        }
    }
}

void ReadAugmentParamsFromIni(const IniFile::Section &section, Zotero::AugmentParams * const augment_params) {
    augment_params->override_ISSN_print_ = section.getString("issn_print", "");
    augment_params->override_ISSN_online_ = section.getString("issn_online", "");
    augment_params->strptime_format_ = section.getString("strptime_format", "");
}

UnsignedPair ProcessRSSFeed(const IniFile::Section &section, const std::shared_ptr<Zotero::HarvestParams> &harvest_params,
                            Zotero::AugmentParams * const augment_params, DbConnection * const db_connection, const bool &test)
{
    ReadAugmentParamsFromIni(section, augment_params);

    const std::string feed_url(section.getString("feed"));
    LOG_DEBUG("feed_url: " + feed_url);
    Zotero::RSSHarvestMode rss_harvest_mode(Zotero::RSSHarvestMode::NORMAL);
    if (test)
        rss_harvest_mode = Zotero::RSSHarvestMode::TEST;
    return Zotero::HarvestSyndicationURL(rss_harvest_mode, feed_url, harvest_params, augment_params, db_connection);
}

void InitSiteDescFromIniFileSection(const IniFile::Section &section, SimpleCrawler::SiteDesc * const site_desc) {
    site_desc->start_url_ = section.getString("base_url");
    site_desc->max_crawl_depth_ = section.getUnsigned("max_crawl_depth");
    site_desc->url_regex_matcher_.reset(RegexMatcher::RegexMatcherFactoryOrDie(section.getString("extraction_regex")));
    site_desc->strptime_format_ = section.getString("strptime_format", "");
}

UnsignedPair ProcessCrawl(const IniFile::Section &section, const std::shared_ptr<Zotero::HarvestParams> &harvest_params,
                          Zotero::AugmentParams * const augment_params, const SimpleCrawler::Params &crawler_params,
                          const std::shared_ptr<RegexMatcher> &supported_urls_regex)
{
    ReadAugmentParamsFromIni(section, augment_params);

    SimpleCrawler::SiteDesc site_desc;
    InitSiteDescFromIniFileSection(section, &site_desc);
    return Zotero::HarvestSite(site_desc, crawler_params, supported_urls_regex, harvest_params, augment_params);
}

UnsignedPair ProcessDirectHarvest(const IniFile::Section &section, const std::shared_ptr<Zotero::HarvestParams> &harvest_params,
                                  Zotero::AugmentParams * const augment_params)
{
    ReadAugmentParamsFromIni(section, augment_params);
    return Zotero::HarvestURL(section.getString("url"), harvest_params, augment_params);
}

std::string GetMarcFormat(const std::string &output_filename) {
    switch (MARC::GuessFileType(output_filename)) {
    case MARC::FileType::BINARY:
        return "marc21";
    case MARC::FileType::XML:
        return "marcxml";
    default:
        LOG_ERROR("can't determine output format from MARC output filename \"" + output_filename + "\"!");
    }
}

} // unnamed namespace

int Main(int argc, char *argv[]) {
    if (argc < 2)
        Usage();

    bool test(false);
    if (std::strcmp(argv[1], "--test") == 0) {
        test = true;
        --argc, ++argv;
    }

    bool ignore_robots_dot_txt(false);
    if (std::strcmp(argv[1], "--ignore-robots-dot-txt") == 0) {
        ignore_robots_dot_txt = true;
        --argc, ++argv;
    }

    std::string map_directory_path;
    const std::string MAP_DIRECTORY_FLAG_PREFIX("--map-directory=");
    if (StringUtil::StartsWith(argv[1], MAP_DIRECTORY_FLAG_PREFIX)) {
        map_directory_path = argv[1] + MAP_DIRECTORY_FLAG_PREFIX.length();
        --argc, ++argv;
    }

    std::string previous_downloads_db_path;
    const std::string PREVIOUS_DOWNLOADS_DB_FLAG_PREFIX("--previous-downloads-db-file=");
    if (StringUtil::StartsWith(argv[1], PREVIOUS_DOWNLOADS_DB_FLAG_PREFIX)) {
        previous_downloads_db_path = argv[1] + PREVIOUS_DOWNLOADS_DB_FLAG_PREFIX.length();
        --argc, ++argv;
    }

    std::string output_file;
    const std::string OUTPUT_FILE_FLAG_PREFIX("--output-file=");
    if (StringUtil::StartsWith(argv[1], OUTPUT_FILE_FLAG_PREFIX)) {
        output_file = argv[1] + OUTPUT_FILE_FLAG_PREFIX.length();
        --argc, ++argv;
    }

    if (argc < 2)
        Usage();

    IniFile ini_file(argv[1]);

    std::shared_ptr<Zotero::HarvestParams> harvest_params(new Zotero::HarvestParams);
    harvest_params->zts_server_url_ = Url(ini_file.getString("", "zts_server_url"));

    if (map_directory_path.empty())
        map_directory_path = ini_file.getString("", "map_directory_path");
    if (previous_downloads_db_path.empty())
        previous_downloads_db_path = ini_file.getString("", "previous_downloads_db_path");

    // ZoteroFormatHandler expects a directory path with a trailing /
    if (not StringUtil::EndsWith(map_directory_path, '/'))
        map_directory_path += "/";

    Zotero::AugmentMaps augment_maps(map_directory_path);
    const std::shared_ptr<RegexMatcher> supported_urls_regex(Zotero::LoadSupportedURLsRegex(map_directory_path));

    std::unique_ptr<DbConnection> db_connection;
    const IniFile rss_ini_file(DbConnection::DEFAULT_CONFIG_FILE_PATH);
    db_connection.reset(new DbConnection(rss_ini_file));

    if (output_file.empty())
        output_file = ini_file.getString("", "marc_output_file");
    harvest_params->format_handler_ = Zotero::FormatHandler::Factory(previous_downloads_db_path, GetMarcFormat(output_file),
                                                                     output_file, harvest_params);

    SimpleCrawler::Params crawler_params;
    crawler_params.ignore_robots_dot_txt_ = ignore_robots_dot_txt;
    crawler_params.min_url_processing_time_ = Zotero::DEFAULT_MIN_URL_PROCESSING_TIME;
    crawler_params.timeout_ = Zotero::DEFAULT_TIMEOUT;

    std::unordered_map<std::string, bool> section_name_to_found_flag_map;
    for (int arg_no(2); arg_no < argc; ++arg_no)
        section_name_to_found_flag_map.emplace(argv[arg_no], false);

    enum Type { RSS, CRAWL, DIRECT };
    const std::map<std::string, int> string_to_value_map{ {"RSS", RSS }, { "CRAWL", CRAWL }, { "DIRECT", DIRECT } };

    unsigned processed_section_count(0);
    UnsignedPair total_record_count_and_previously_downloaded_record_count;

    for (const auto &section : ini_file) {
        if (section.first.empty())
            continue;       // don't parse the global parameters section

        std::vector<MARC::EditInstruction> edit_instructions;
        LoadMARCEditInstructions(section.second, &edit_instructions);

        Zotero::AugmentParams augment_params(&augment_maps, edit_instructions);
        harvest_params->format_handler_->setAugmentParams(&augment_params);

        if (not section_name_to_found_flag_map.empty()) {
            const auto section_name_and_found_flag(section_name_to_found_flag_map.find(section.first));
            if (section_name_and_found_flag == section_name_to_found_flag_map.end())
                continue;
            section_name_and_found_flag->second = true;
        }
        ++processed_section_count;

        LOG_INFO("Processing section \"" + section.first + "\".");
        const Type type(static_cast<Type>(section.second.getEnum("type", string_to_value_map)));
        if (type == RSS)
            total_record_count_and_previously_downloaded_record_count += ProcessRSSFeed(section.second, harvest_params, &augment_params,
                                                                                db_connection.get(), test);
        else if (type == CRAWL)
            total_record_count_and_previously_downloaded_record_count +=
                ProcessCrawl(section.second, harvest_params, &augment_params, crawler_params, supported_urls_regex);
        else
            total_record_count_and_previously_downloaded_record_count +=
                ProcessDirectHarvest(section.second, harvest_params, &augment_params);
    }

    LOG_INFO("Extracted metadata from "
             + std::to_string(total_record_count_and_previously_downloaded_record_count.first
                              - total_record_count_and_previously_downloaded_record_count.second) + " page(s).");

    if (section_name_to_found_flag_map.size() > processed_section_count) {
        std::cerr << "The following sections were specified but not processed:\n";
        for (const auto &section_name_and_found_flag : section_name_to_found_flag_map)
            std::cerr << '\t' << section_name_and_found_flag.first << '\n';
    }

    return EXIT_SUCCESS;
}
<commit_msg>Added support for tagging sections as "live".<commit_after>/** \brief Handles crawling as well as RSS feeds.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2018 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <iostream>
#include <memory>
#include <unordered_map>
#include <cstdlib>
#include "DbConnection.h"
#include "IniFile.h"
#include "MARC.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "util.h"
#include "Zotero.h"

namespace {

[[noreturn]] void Usage() {
    std::cerr << "Usage: " << ::progname << " [options] config_file_path [section1 section2 .. sectionN]\n"
              << "\n"
              << "\tOptions:\n"
              << "\t[--verbosity=log_level]        Possible log levels are ERROR, WARNING, INFO, and DEBUG with the default being WARNING.\n"
              << "\t[--test]                       No download information will be stored for further downloads.\n"
              << "\t[--live-only]                  Only sections that have \"live=true\" set will be processed.\n"
              << "\t[--ignore-robots-dot-txt]\n"
              << "\t[--map-directory=map_directory]\n"
              << "\t[--previous-downloads-db-file=previous_downloads_db_file]\n"
              << "\t[--output-file=output_file]\n"
              << "\n"
              << "\tIf any section names have been provided, only those will be processed o/w all sections will be processed.\n\n";
    std::exit(EXIT_FAILURE);
}

void LoadMARCEditInstructions(const IniFile::Section &section, std::vector<MARC::EditInstruction> * edit_instructions) {
    edit_instructions->clear();

    for (const auto &name_and_value : section) {
        if (StringUtil::StartsWith(name_and_value.first, "insert_field_")) {
            const std::string tag_candidate(name_and_value.first.substr(__builtin_strlen("insert_field_")));
            if (tag_candidate.length() != MARC::Record::TAG_LENGTH)
                LOG_ERROR("bad entry in section \"" + section.getSectionName() + "\" \"" + name_and_value.first + "\"!");
            edit_instructions->emplace_back(MARC::EditInstruction::CreateInsertFieldInstruction(tag_candidate, name_and_value.second));
        } else if (StringUtil::StartsWith(name_and_value.first, "insert_subfield_")) {
            const std::string tag_and_subfield_code_candidate(name_and_value.first.substr(__builtin_strlen("insert_subfield_")));
            if (tag_and_subfield_code_candidate.length() != MARC::Record::TAG_LENGTH + 1)
                LOG_ERROR("bad entry in section \"" + section.getSectionName() + "\" \"" + name_and_value.first + "\"!");
            edit_instructions->emplace_back(MARC::EditInstruction::CreateInsertSubfieldInstruction(
                tag_and_subfield_code_candidate.substr(0, MARC::Record::TAG_LENGTH),
                tag_and_subfield_code_candidate[MARC::Record::TAG_LENGTH], name_and_value.second));
        } else if (StringUtil::StartsWith(name_and_value.first, "add_subfield_")) {
            const std::string tag_and_subfield_code_candidate(name_and_value.first.substr(__builtin_strlen("add_subfield_")));
            if (tag_and_subfield_code_candidate.length() != MARC::Record::TAG_LENGTH + 1)
                LOG_ERROR("bad entry in section \"" + section.getSectionName() + "\" \"" + name_and_value.first + "\"!");
            edit_instructions->emplace_back(MARC::EditInstruction::CreateAddSubfieldInstruction(
                tag_and_subfield_code_candidate.substr(0, MARC::Record::TAG_LENGTH),
                tag_and_subfield_code_candidate[MARC::Record::TAG_LENGTH], name_and_value.second));
        }
    }
}

void ReadAugmentParamsFromIni(const IniFile::Section &section, Zotero::AugmentParams * const augment_params) {
    augment_params->override_ISSN_print_ = section.getString("issn_print", "");
    augment_params->override_ISSN_online_ = section.getString("issn_online", "");
    augment_params->strptime_format_ = section.getString("strptime_format", "");
}

UnsignedPair ProcessRSSFeed(const IniFile::Section &section, const std::shared_ptr<Zotero::HarvestParams> &harvest_params,
                            Zotero::AugmentParams * const augment_params, DbConnection * const db_connection, const bool &test)
{
    ReadAugmentParamsFromIni(section, augment_params);

    const std::string feed_url(section.getString("feed"));
    LOG_DEBUG("feed_url: " + feed_url);
    Zotero::RSSHarvestMode rss_harvest_mode(Zotero::RSSHarvestMode::NORMAL);
    if (test)
        rss_harvest_mode = Zotero::RSSHarvestMode::TEST;
    return Zotero::HarvestSyndicationURL(rss_harvest_mode, feed_url, harvest_params, augment_params, db_connection);
}

void InitSiteDescFromIniFileSection(const IniFile::Section &section, SimpleCrawler::SiteDesc * const site_desc) {
    site_desc->start_url_ = section.getString("base_url");
    site_desc->max_crawl_depth_ = section.getUnsigned("max_crawl_depth");
    site_desc->url_regex_matcher_.reset(RegexMatcher::RegexMatcherFactoryOrDie(section.getString("extraction_regex")));
    site_desc->strptime_format_ = section.getString("strptime_format", "");
}

UnsignedPair ProcessCrawl(const IniFile::Section &section, const std::shared_ptr<Zotero::HarvestParams> &harvest_params,
                          Zotero::AugmentParams * const augment_params, const SimpleCrawler::Params &crawler_params,
                          const std::shared_ptr<RegexMatcher> &supported_urls_regex)
{
    ReadAugmentParamsFromIni(section, augment_params);

    SimpleCrawler::SiteDesc site_desc;
    InitSiteDescFromIniFileSection(section, &site_desc);
    return Zotero::HarvestSite(site_desc, crawler_params, supported_urls_regex, harvest_params, augment_params);
}

UnsignedPair ProcessDirectHarvest(const IniFile::Section &section, const std::shared_ptr<Zotero::HarvestParams> &harvest_params,
                                  Zotero::AugmentParams * const augment_params)
{
    ReadAugmentParamsFromIni(section, augment_params);
    return Zotero::HarvestURL(section.getString("url"), harvest_params, augment_params);
}

std::string GetMarcFormat(const std::string &output_filename) {
    switch (MARC::GuessFileType(output_filename)) {
    case MARC::FileType::BINARY:
        return "marc21";
    case MARC::FileType::XML:
        return "marcxml";
    default:
        LOG_ERROR("can't determine output format from MARC output filename \"" + output_filename + "\"!");
    }
}

} // unnamed namespace

int Main(int argc, char *argv[]) {
    if (argc < 2)
        Usage();

    bool test(false);
    if (std::strcmp(argv[1], "--test") == 0) {
        test = true;
        --argc, ++argv;
    }

    bool live_only(false);
    if (std::strcmp(argv[1], "--live-only") == 0) {
        live_only = true;
        --argc, ++argv;
    }

    bool ignore_robots_dot_txt(false);
    if (std::strcmp(argv[1], "--ignore-robots-dot-txt") == 0) {
        ignore_robots_dot_txt = true;
        --argc, ++argv;
    }

    std::string map_directory_path;
    const std::string MAP_DIRECTORY_FLAG_PREFIX("--map-directory=");
    if (StringUtil::StartsWith(argv[1], MAP_DIRECTORY_FLAG_PREFIX)) {
        map_directory_path = argv[1] + MAP_DIRECTORY_FLAG_PREFIX.length();
        --argc, ++argv;
    }

    std::string previous_downloads_db_path;
    const std::string PREVIOUS_DOWNLOADS_DB_FLAG_PREFIX("--previous-downloads-db-file=");
    if (StringUtil::StartsWith(argv[1], PREVIOUS_DOWNLOADS_DB_FLAG_PREFIX)) {
        previous_downloads_db_path = argv[1] + PREVIOUS_DOWNLOADS_DB_FLAG_PREFIX.length();
        --argc, ++argv;
    }

    std::string output_file;
    const std::string OUTPUT_FILE_FLAG_PREFIX("--output-file=");
    if (StringUtil::StartsWith(argv[1], OUTPUT_FILE_FLAG_PREFIX)) {
        output_file = argv[1] + OUTPUT_FILE_FLAG_PREFIX.length();
        --argc, ++argv;
    }

    if (argc < 2)
        Usage();

    IniFile ini_file(argv[1]);

    std::shared_ptr<Zotero::HarvestParams> harvest_params(new Zotero::HarvestParams);
    harvest_params->zts_server_url_ = Url(ini_file.getString("", "zts_server_url"));

    if (map_directory_path.empty())
        map_directory_path = ini_file.getString("", "map_directory_path");
    if (previous_downloads_db_path.empty())
        previous_downloads_db_path = ini_file.getString("", "previous_downloads_db_path");

    // ZoteroFormatHandler expects a directory path with a trailing /
    if (not StringUtil::EndsWith(map_directory_path, '/'))
        map_directory_path += "/";

    Zotero::AugmentMaps augment_maps(map_directory_path);
    const std::shared_ptr<RegexMatcher> supported_urls_regex(Zotero::LoadSupportedURLsRegex(map_directory_path));

    std::unique_ptr<DbConnection> db_connection;
    const IniFile rss_ini_file(DbConnection::DEFAULT_CONFIG_FILE_PATH);
    db_connection.reset(new DbConnection(rss_ini_file));

    if (output_file.empty())
        output_file = ini_file.getString("", "marc_output_file");
    harvest_params->format_handler_ = Zotero::FormatHandler::Factory(previous_downloads_db_path, GetMarcFormat(output_file),
                                                                     output_file, harvest_params);

    SimpleCrawler::Params crawler_params;
    crawler_params.ignore_robots_dot_txt_ = ignore_robots_dot_txt;
    crawler_params.min_url_processing_time_ = Zotero::DEFAULT_MIN_URL_PROCESSING_TIME;
    crawler_params.timeout_ = Zotero::DEFAULT_TIMEOUT;

    std::unordered_map<std::string, bool> section_name_to_found_flag_map;
    for (int arg_no(2); arg_no < argc; ++arg_no)
        section_name_to_found_flag_map.emplace(argv[arg_no], false);

    enum Type { RSS, CRAWL, DIRECT };
    const std::map<std::string, int> string_to_value_map{ {"RSS", RSS }, { "CRAWL", CRAWL }, { "DIRECT", DIRECT } };

    unsigned processed_section_count(0);
    UnsignedPair total_record_count_and_previously_downloaded_record_count;

    for (const auto &section : ini_file) {
        if (section.first.empty())
            continue;       // don't parse the global parameters section

        if (live_only and section.second.getBool("live", false) != true)
            continue;

        std::vector<MARC::EditInstruction> edit_instructions;
        LoadMARCEditInstructions(section.second, &edit_instructions);

        Zotero::AugmentParams augment_params(&augment_maps, edit_instructions);
        harvest_params->format_handler_->setAugmentParams(&augment_params);

        if (not section_name_to_found_flag_map.empty()) {
            const auto section_name_and_found_flag(section_name_to_found_flag_map.find(section.first));
            if (section_name_and_found_flag == section_name_to_found_flag_map.end())
                continue;
            section_name_and_found_flag->second = true;
        }
        ++processed_section_count;

        LOG_INFO("Processing section \"" + section.first + "\".");
        const Type type(static_cast<Type>(section.second.getEnum("type", string_to_value_map)));
        if (type == RSS)
            total_record_count_and_previously_downloaded_record_count += ProcessRSSFeed(section.second, harvest_params, &augment_params,
                                                                                db_connection.get(), test);
        else if (type == CRAWL)
            total_record_count_and_previously_downloaded_record_count +=
                ProcessCrawl(section.second, harvest_params, &augment_params, crawler_params, supported_urls_regex);
        else
            total_record_count_and_previously_downloaded_record_count +=
                ProcessDirectHarvest(section.second, harvest_params, &augment_params);
    }

    LOG_INFO("Extracted metadata from "
             + std::to_string(total_record_count_and_previously_downloaded_record_count.first
                              - total_record_count_and_previously_downloaded_record_count.second) + " page(s).");

    if (section_name_to_found_flag_map.size() > processed_section_count) {
        std::cerr << "The following sections were specified but not processed:\n";
        for (const auto &section_name_and_found_flag : section_name_to_found_flag_map)
            std::cerr << '\t' << section_name_and_found_flag.first << '\n';
    }

    return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>/* symtable.cc
 * Hash table representing the symbol table.
 * the symbol table.
 * UIdaho CS445 120++ Compiler
 * author: Chris Waltrip <walt2178@vandals.uidaho.edu>
 */
#include <iostream>
#include <sstream>
#include <string>

#include "symtable.hh"
#include "exception.hh"

/*
AbstractSymbol::AbstractSymbol(std::string n, TypenameEntry t)
			: name(n), type(t) { }
*/
AbstractSymbol::AbstractSymbol(std::string n, std::string t)
			: name(n), type(t) { }
std::string AbstractSymbol::to_string(std::size_t depth) {
	std::string spaces = std::string(depth, '>');
	return(spaces + this->type + " " + this->name);
}

/*
 * The only time that SymbolTable's parent will be NULL is the Global Symbol
 * Table.  Given this and the n-ary tree (represented using std::deque),
 * resolving scope should be relatively simple.
 */
SymbolTable::SymbolTable(SymbolTable *p) {
	this->parent = p;
}

/*
 * Just a wrapper function for nesting scopes.
 */
void SymbolTable::add_sub_table(SymbolTable *k) {
	this->kids.push_back(k);
}
/*
 * This hashing method is identical to the Typename Table hash method.
 */
std::size_t SymbolTable::hash(std::string name) {
	std::size_t hash = 0;
	const char* s = name.c_str();
	while(*s) {
		hash = hash * 101 + *s++;
	}
	return hash % this->HASHTABLE_SIZE;
}

bool SymbolTable::insert(std::string n, AbstractSymbol *s) {
	if(!this->symbol_exists(n)) {
		std::size_t h = this->hash(n);
		this->bucket[h].push_back(s);
		return true;
	}
	throw EDuplicateSymbol();
	return false; /* Remove after refactor */
}

bool SymbolTable::empty() {
	for(std::size_t i = 0; i < this->HASHTABLE_SIZE; ++i) {
		if(!this->bucket[i].empty())
			return false;
	}
	return true;
}

AbstractSymbol* SymbolTable::get_symbol(std::string name) {
	std::size_t h = this->hash(name);
	std::deque<AbstractSymbol*> b = this->bucket[h];
	std::deque<AbstractSymbol*>::iterator it;
	for(it = b.begin(); it != b.end(); ++it) {
		std::size_t i = it - b.begin();
		if(*(b[i]) == name) {
			return b[i];
		}
	}
	throw ENoSymbolEntry();
}

/*
 * Tries to get the symbol from the current scope.  If get_symbol throws an
 * ENoSymbolEntry, then the current scope doesn't have the symbol that is
 * being search for.  If the parent symbol table is NULL then we have reached
 * the global symbol table and the symbol doesn't exist so throw
 * ENoSymbolEntry again and let the calling function handle the exception. If
 * the symbol is found, it's returned.
 */
AbstractSymbol* SymbolTable::get_scoped_symbol(std::string name) {
	AbstractSymbol *symb;
	try {
		symb = this->get_symbol(name);
		return symb;
	} catch(ENoSymbolEntry e) {
		if(this->parent == NULL) {
			throw ENoSymbolEntry();
		} else {
			/* Will never be NULL because exception is thrown */
			return this->parent->get_scoped_symbol(name);
		}
	}
}

/*
 * Will return true if a symbol is found.  If get_symbol throws
 * an ENoSymbolEntry exception, then the symbol isn't in the symbol
 * table.  This only checks for the existance in the local symbol table.
 */
bool SymbolTable::symbol_exists(std::string name) {
	try {
		this->get_symbol(name);
	} catch(ENoSymbolEntry e) {
		return false;
	}
	return true;
}

void SymbolTable::print_table(std::size_t depth) {
	std::clog << this->to_string(depth) << std::endl;
}

std::string SymbolTable::to_string(std::size_t depth) {
	std::stringstream ss;
	for(std::size_t i = 0; i < this->HASHTABLE_SIZE; ++i) {
		std::deque<AbstractSymbol*> b = this->bucket[i];
		std::deque<AbstractSymbol*>::iterator it;
		for(it = b.begin(); it != b.end(); ++it) {
			ss << (*it)->to_string(depth) << std::endl;
		}
	}
	return ss.str();
}

/* Basic Symbol constructor */
BasicSymbol::BasicSymbol(std::string n, std::string t, bool p)
				: AbstractSymbol(n,t), pointer(p) { }

/* Function symbol constructor */
FunctionSymbol::FunctionSymbol(std::string n, std::string t, bool p, bool d)
				: AbstractSymbol(n,t), pointer(p),
				def_needed(d) { }

std::string FunctionSymbol::to_string(std::size_t depth) {
	std::stringstream ss;
	ss << AbstractSymbol::to_string(depth) << std::endl;
	ss << this->params.to_string(depth+1) << std::endl;
	ss << this->locals.to_string(depth+1) << std::endl;
	return ss.str();
}<commit_msg>Fixing segfault when referencing iterator<commit_after>/* symtable.cc
 * Hash table representing the symbol table.
 * the symbol table.
 * UIdaho CS445 120++ Compiler
 * author: Chris Waltrip <walt2178@vandals.uidaho.edu>
 */
#include <iostream>
#include <sstream>
#include <string>

#include "symtable.hh"
#include "exception.hh"

/*
AbstractSymbol::AbstractSymbol(std::string n, TypenameEntry t)
			: name(n), type(t) { }
*/
AbstractSymbol::AbstractSymbol(std::string n, std::string t)
			: name(n), type(t) { }
std::string AbstractSymbol::to_string(std::size_t depth) {
	std::string spaces = std::string(depth, '>');
	return(spaces + this->type + " " + this->name);
}

/*
 * The only time that SymbolTable's parent will be NULL is the Global Symbol
 * Table.  Given this and the n-ary tree (represented using std::deque),
 * resolving scope should be relatively simple.
 */
SymbolTable::SymbolTable(SymbolTable *p) {
	this->parent = p;
}

/*
 * Just a wrapper function for nesting scopes.
 */
void SymbolTable::add_sub_table(SymbolTable *k) {
	this->kids.push_back(k);
}
/*
 * This hashing method is identical to the Typename Table hash method.
 */
std::size_t SymbolTable::hash(std::string name) {
	std::size_t hash = 0;
	const char* s = name.c_str();
	while(*s) {
		hash = hash * 101 + *s++;
	}
	return hash % this->HASHTABLE_SIZE;
}

bool SymbolTable::insert(std::string n, AbstractSymbol *s) {
	if(!this->symbol_exists(n)) {
		std::size_t h = this->hash(n);
		this->bucket[h].push_back(s);
		return true;
	}
	throw EDuplicateSymbol();
	return false; /* Remove after refactor */
}

bool SymbolTable::empty() {
	for(std::size_t i = 0; i < this->HASHTABLE_SIZE; ++i) {
		if(!this->bucket[i].empty())
			return false;
	}
	return true;
}

AbstractSymbol* SymbolTable::get_symbol(std::string name) {
	std::size_t h = this->hash(name);
	std::deque<AbstractSymbol*> b = this->bucket[h];
	std::deque<AbstractSymbol*>::iterator it;
	for(it = b.begin(); it != b.end(); ++it) {
		std::size_t i = it - b.begin();
		if(*(b[i]) == name) {
			return b[i];
		}
	}
	throw ENoSymbolEntry();
}

/*
 * Tries to get the symbol from the current scope.  If get_symbol throws an
 * ENoSymbolEntry, then the current scope doesn't have the symbol that is
 * being search for.  If the parent symbol table is NULL then we have reached
 * the global symbol table and the symbol doesn't exist so throw
 * ENoSymbolEntry again and let the calling function handle the exception. If
 * the symbol is found, it's returned.
 */
AbstractSymbol* SymbolTable::get_scoped_symbol(std::string name) {
	AbstractSymbol *symb;
	try {
		symb = this->get_symbol(name);
		return symb;
	} catch(ENoSymbolEntry e) {
		if(this->parent == NULL) {
			throw ENoSymbolEntry();
		} else {
			/* Will never be NULL because exception is thrown */
			return this->parent->get_scoped_symbol(name);
		}
	}
}

/*
 * Will return true if a symbol is found.  If get_symbol throws
 * an ENoSymbolEntry exception, then the symbol isn't in the symbol
 * table.  This only checks for the existance in the local symbol table.
 */
bool SymbolTable::symbol_exists(std::string name) {
	try {
		this->get_symbol(name);
	} catch(ENoSymbolEntry e) {
		return false;
	}
	return true;
}

void SymbolTable::print_table(std::size_t depth) {
	std::clog << this->to_string(depth) << std::endl;
}

std::string SymbolTable::to_string(std::size_t depth) {
	std::stringstream ss;
	for(std::size_t i = 0; i < this->HASHTABLE_SIZE; ++i) {
		std::deque<AbstractSymbol*> b = this->bucket[i];
		std::deque<AbstractSymbol*>::iterator it;
		for(it = b.begin(); it != b.end(); ++it) {\
			std::size_t index = it - b.begin();
			ss << (*(b[i]))->to_string(depth) << std::endl;
			//ss << (*it)->to_string(depth) << std::endl;
		}
	}
	return ss.str();
}

/* Basic Symbol constructor */
BasicSymbol::BasicSymbol(std::string n, std::string t, bool p)
				: AbstractSymbol(n,t), pointer(p) { }

/* Function symbol constructor */
FunctionSymbol::FunctionSymbol(std::string n, std::string t, bool p, bool d)
				: AbstractSymbol(n,t), pointer(p),
				def_needed(d) { }

std::string FunctionSymbol::to_string(std::size_t depth) {
	std::stringstream ss;
	ss << AbstractSymbol::to_string(depth) << std::endl;
	ss << this->params.to_string(depth+1) << std::endl;
	ss << this->locals.to_string(depth+1) << std::endl;
	return ss.str();
}<|endoftext|>


<commit_before>#include "cqts_classviewer.h"
#include <QLabel>
#include <QComboBox>

CQTs_ClassViewer::CQTs_ClassViewer(QWidget *parent) :
    QGroupBox(tr("Class Viewer"),parent)
{
    QLabel Sel(tr("Class:"));
    QComboBox ComboSel();

}
<commit_msg>Interface for classviewer done<commit_after>#include "cqts_classviewer.h"
#include <QLabel>
#include <QComboBox>
#include <QGridLayout>
CQTs_ClassViewer::CQTs_ClassViewer(QWidget *parent) :
    QGroupBox(tr("Class Viewer"),parent)
{
    QGridLayout *grid = new QGridLayout();

    QLabel *Tlab = new QLabel(tr("Class:"));
    grid->addWidget(Tlab,0,0);

    Tlab = new QLabel(tr("Base Attack bonus:"));
    grid->addWidget(Tlab,1,0);

    QGroupBox *SavesBox= new QGroupBox(tr("Saves:"));
    grid->addWidget(SavesBox,2,0,1,2);
    QGridLayout *grid2 = new QGridLayout();
    Tlab = new QLabel(tr("Fortitude:"));
    grid2->addWidget(Tlab,0,0);
    Tlab = new QLabel(tr("Reflexes"));
    grid2->addWidget(Tlab,1,0);
    Tlab = new QLabel(tr("Will:"));
    grid2->addWidget(Tlab,2,0);
    SavesBox->setLayout(grid2);

    QComboBox *ComboSel = new QComboBox();
    grid->addWidget(ComboSel,0,1);

    setLayout(grid);
}
<|endoftext|>


<commit_before><commit_msg>add TDPC_D<commit_after><|endoftext|>


<commit_before>// Generates a series of topologies from a set of locations. Each of
// the topologies will have a progressively higher level of connectivity,
// starting from no connectivity and ending at a full clique. Each link will
// have a delay set up to be the speed of light in fiber.

#include <gflags/gflags.h>
#include <stddef.h>
#include <stdint.h>
#include <algorithm>
#include <chrono>
#include <limits>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <tuple>
#include <utility>
#include <vector>

#include "ncode_common/src/common.h"
#include "ncode_common/src/file.h"
#include "ncode_common/src/logging.h"
#include "ncode_common/src/lp/demand_matrix.h"
#include "ncode_common/src/map_util.h"
#include "ncode_common/src/net/net_common.h"
#include "ncode_common/src/net/net_gen.h"
#include "ncode_common/src/strutil.h"
#include "ncode_common/src/thread_runner.h"
#include "ncode_common/src/viz/web_page.h"
#include "common.h"
#include "geo/geo.h"
#include "metrics/metrics.h"
#include "opt/ctr.h"
#include "opt/opt.h"
#include "opt/path_provider.h"

DEFINE_string(cities_file, "cities5000.txt", "Location of the cities file");
DEFINE_string(topology_file, "", "Topology file");
DEFINE_string(tm_file, "", "Traffic matrix file");
DEFINE_double(link_capacity_scale, 1.0, "By how much to scale all links");
DEFINE_double(delay_scale, 1.0, "By how much to scale the delays of all links");
DEFINE_double(
    light_speed, 200.0,
    "Speed of light (in km per millisecond), default is speed in fiber");
DEFINE_double(link_speed_Mbps, 1000, "All new links will be this fast");
DEFINE_double(scale_step, 1.01, "By how much to scale the TM each step");
DEFINE_double(capacity_grow_threshold, 1.3,
              "What the limit is for growing capacity");
DEFINE_string(
    optimizer, "CTR",
    "The optimizer to use. One of SP,CTR,MinMax,MinMaxLD,MinMaxK10,B4.");
DEFINE_uint64(threads, 4, "Number of threads to use");

using namespace std::chrono;

// Elements of a traffic matrix.
using TMElements = std::vector<nc::lp::DemandMatrixElement>;

// Number of flows in the most loaded aggregate.
static constexpr size_t kTopAggregateFlowCount = 10000;

static auto* total_delay_ms =
    nc::metrics::DefaultMetricManager()
        -> GetThreadSafeMetric<uint32_t, uint32_t>("total_delay_ms",
                                                   "Sum of all delays", "Step");

static auto* max_link_utilization =
    nc::metrics::DefaultMetricManager()
        -> GetThreadSafeMetric<double, uint32_t>(
            "max_link_utilization", "Maximum link utilization", "Step");

namespace ctr {

static std::unique_ptr<TrafficMatrix> FromDemandMatrix(
    const nc::lp::DemandMatrix& demand_matrix) {
  nc::net::Bandwidth max_demand = nc::net::Bandwidth::Zero();
  for (const auto& element : demand_matrix.elements()) {
    max_demand = std::max(max_demand, element.demand);
  }

  // Will assign flow counts so that the max bandwidth aggregate has
  // kTopAggregateFlowCount, and all other aggregates proportionally less.
  std::map<AggregateId, DemandAndFlowCount> demands_and_counts;
  for (const auto& element : demand_matrix.elements()) {
    size_t flow_count = kTopAggregateFlowCount * (element.demand / max_demand);
    flow_count = std::max(1ul, flow_count);

    AggregateId id(element.src, element.dst);
    demands_and_counts[id] = {element.demand, flow_count};
  }

  return nc::make_unique<TrafficMatrix>(demand_matrix.graph(),
                                        demands_and_counts);
}

static std::unique_ptr<RoutingConfiguration> RunOptimizer(
    const nc::lp::DemandMatrix& demand_matrix, PathProvider* provider) {
  auto tm = FromDemandMatrix(demand_matrix);

  std::unique_ptr<Optimizer> opt;
  if (FLAGS_optimizer == "CTR") {
    opt = nc::make_unique<CTROptimizer>(provider, 1.0, false, false);
  } else if (FLAGS_optimizer == "MinMax") {
    opt = nc::make_unique<MinMaxOptimizer>(provider, 1.0, false);
  } else if (FLAGS_optimizer == "MinMaxLD") {
    opt = nc::make_unique<MinMaxOptimizer>(provider, 1.0, true);
  } else if (FLAGS_optimizer == "MinMaxK10") {
    opt = nc::make_unique<MinMaxPathBasedOptimizer>(provider, 1.0, true, 10);
  } else if (FLAGS_optimizer == "B4") {
    opt = nc::make_unique<B4Optimizer>(provider, false, 1.0);
  } else if (FLAGS_optimizer == "SP") {
    opt = nc::make_unique<ShortestPathOptimizer>(provider);
  } else {
    LOG(FATAL) << "Bad optimizer " << FLAGS_optimizer;
  }

  return opt->Optimize(*tm);
}

// Scores a combination of topology and TM.
static std::pair<double, bool> Cost(const nc::net::GraphBuilder& topology,
                                    const TMElements& tm_elements) {
  nc::net::GraphStorage graph(topology);
  PathProvider path_provider(&graph);
  nc::lp::DemandMatrix demand_matrix(tm_elements, &graph);
  auto routing = RunOptimizer(demand_matrix, &path_provider);

  double total_oversubscription = 0;
  double max_utilization = 0;
  nc::net::GraphLinkMap<double> link_utilizations = routing->LinkUtilizations();
  for (const auto& link_and_utilization : link_utilizations) {
    double utilization = *(link_and_utilization.second);

    double oversubscription = utilization - 1.0;
    oversubscription = std::max(0.0, oversubscription);
    total_oversubscription += oversubscription;

    max_utilization = std::max(max_utilization, utilization);
  }

  return {total_oversubscription, max_utilization <= 1.0};
}

// Records information about the routing.
static void RecordRoutingConfig(const RoutingConfiguration& routing,
                                uint32_t step) {
  auto* total_path_handle = total_delay_ms->GetHandle(step);
  auto* link_utilization_handle = max_link_utilization->GetHandle(step);

  nc::net::Delay total_delay = routing.TotalPerFlowDelay();
  total_path_handle->AddValue(duration_cast<milliseconds>(total_delay).count());

  double max_utilization = routing.MaxLinkUtilization();
  link_utilization_handle->AddValue(max_utilization);

  nc::viz::HtmlPage page;
  routing.ToHTML(&page);
  nc::File::WriteStringToFile(page.Construct(),
                              nc::StrCat("top_grow_step_", step, ".html"));
}

// Will add a single link (or increase the capacity of existing one) that
// minimizes the overall cost.
static nc::net::GraphBuilder GrowNetwork(
    const nc::net::GraphBuilder& topology, const TMElements& tm_elements,
    const std::map<std::pair<std::string, std::string>, milliseconds>& delays) {
  nc::net::GraphBuilder to_return = topology;
  while (true) {
    nc::net::GraphBuilder best_topology;
    double best_cost = std::numeric_limits<double>::max();
    bool best_fits = false;
    std::string best_link_to_string;

    std::vector<nc::net::GraphBuilder> to_run;
    std::vector<std::string> links_added_to_string;
    std::set<std::string> all_nodes = topology.AllNodeNames();
    for (const std::string& src : all_nodes) {
      for (const std::string& dst : all_nodes) {
        if (src >= dst) {
          continue;
        }

        nc::net::GraphBuilder new_topology = to_return;
        nc::net::Delay delay =
            nc::FindOrDieNoPrint(delays, std::make_pair(src, dst));
        nc::net::Bandwidth bw =
            nc::net::Bandwidth::FromMBitsPerSecond(FLAGS_link_speed_Mbps);
        new_topology.AddLink({src, dst, bw, delay});
        new_topology.AddLink({dst, src, bw, delay});
        new_topology.RemoveMultipleLinks();
        to_run.emplace_back(new_topology);
        links_added_to_string.emplace_back(nc::StrCat(src, "->", dst));
      }
    }

    std::vector<std::unique_ptr<std::pair<double, bool>>> output =
        nc::RunInParallelWithResult<nc::net::GraphBuilder,
                                    std::pair<double, bool>>(
            to_run, [&tm_elements](const nc::net::GraphBuilder& builder) {
              return nc::make_unique<std::pair<double, bool>>(
                  Cost(builder, tm_elements));
            }, FLAGS_threads);

    for (size_t i = 0; i < to_run.size(); ++i) {
      const nc::net::GraphBuilder& new_topology = to_run[i];
      double cost;
      bool fits;
      std::tie(cost, fits) = *(output[i]);

      if (cost < best_cost) {
        best_topology = new_topology;
        best_cost = cost;
        best_link_to_string = links_added_to_string[i];
        best_fits = fits;
      }
    }

    CHECK(best_cost != std::numeric_limits<double>::max());
    LOG(INFO) << "Growing network, best cost " << best_cost << " will add "
              << best_link_to_string << " fits " << best_fits;

    to_return = best_topology;
    if (best_fits) {
      break;
    }
  }

  return to_return;
}

static void Run(
    const nc::net::GraphBuilder& topology, const TMElements& tm_elements,
    const std::map<std::pair<std::string, std::string>, milliseconds>& delays) {
  nc::net::GraphBuilder current_topology = topology;
  TMElements current_elements = tm_elements;

  for (size_t i = 0; i < 100; ++i) {
    LOG(INFO) << "Step " << i;

    // Will first run to record information.
    nc::net::GraphStorage graph(current_topology);
    PathProvider path_provider(&graph);

    nc::lp::DemandMatrix demand_matrix(current_elements, &graph);
    auto output = RunOptimizer(demand_matrix, &path_provider);
    RecordRoutingConfig(*output, i);

    // Will grow the TM to figure out if any new links need to be added.
    auto scaled_demands = demand_matrix.Scale(FLAGS_capacity_grow_threshold);
    output = RunOptimizer(*scaled_demands, &path_provider);

    bool all_fit = output->MaxLinkUtilization() <= 1.0;
    if (!all_fit) {
      // Need to grow the network.
      current_topology =
          GrowNetwork(topology, scaled_demands->elements(), delays);
    }

    // Perform a single step.
    scaled_demands = demand_matrix.Scale(FLAGS_scale_step);
    current_elements = scaled_demands->elements();
  }
}

}  // namespace ctr

static const nc::geo::CityData* GetCity(const std::string& name,
                                        nc::geo::Localizer* localizer) {
  nc::geo::FindCityRequest request;
  request.ascii_name = name;

  const nc::geo::CityData* city_data = localizer->FindCityOrNull(request);
  LOG(INFO) << name << " localized to " << city_data->ToString();
  CHECK(city_data != nullptr);
  return city_data;
}

int main(int argc, char** argv) {
  gflags::ParseCommandLineFlags(&argc, &argv, true);
  nc::metrics::InitMetrics();
  nc::geo::Localizer localizer(FLAGS_cities_file);

  std::vector<std::string> node_order;
  nc::net::GraphBuilder builder = nc::net::LoadRepetitaOrDie(
      nc::File::ReadFileToStringOrDie(FLAGS_topology_file), &node_order);
  builder.RemoveMultipleLinks();
  builder.ScaleCapacity(FLAGS_link_capacity_scale);
  builder.ScaleDelay(FLAGS_delay_scale);

  nc::net::GraphStorage graph(builder);
  std::unique_ptr<nc::lp::DemandMatrix> demand_matrix =
      nc::lp::DemandMatrix::LoadRepetitaOrDie(
          nc::File::ReadFileToStringOrDie(FLAGS_tm_file), node_order, &graph);

  // Will record the delays between all N * (N - 1) nodes.
  std::map<std::pair<std::string, std::string>, milliseconds> delays;
  nc::net::GraphNodeSet all_nodes = graph.AllNodes();
  for (nc::net::GraphNodeIndex src : all_nodes) {
    for (nc::net::GraphNodeIndex dst : all_nodes) {
      if (src == dst) {
        continue;
      }

      const std::string& src_id = graph.GetNode(src)->id();
      const std::string& dst_id = graph.GetNode(dst)->id();

      const nc::geo::CityData* src_city = GetCity(src_id, &localizer);
      const nc::geo::CityData* dst_city = GetCity(dst_id, &localizer);
      double distance_km = src_city->DistanceKm(*dst_city);

      uint32_t delay_ms = distance_km / FLAGS_light_speed;
      delay_ms = std::max(delay_ms, static_cast<uint32_t>(1));
      delays[{src_id, dst_id}] = milliseconds(delay_ms);
    }
  }

  ctr::Run(builder, demand_matrix->elements(), delays);
}
<commit_msg>updated top_grow<commit_after>// Generates a series of topologies from a set of locations. Each of
// the topologies will have a progressively higher level of connectivity,
// starting from no connectivity and ending at a full clique. Each link will
// have a delay set up to be the speed of light in fiber.

#include <gflags/gflags.h>
#include <stddef.h>
#include <stdint.h>
#include <algorithm>
#include <chrono>
#include <limits>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <tuple>
#include <utility>
#include <vector>

#include "ncode_common/src/common.h"
#include "ncode_common/src/file.h"
#include "ncode_common/src/logging.h"
#include "ncode_common/src/lp/demand_matrix.h"
#include "ncode_common/src/map_util.h"
#include "ncode_common/src/net/net_common.h"
#include "ncode_common/src/net/net_gen.h"
#include "ncode_common/src/strutil.h"
#include "ncode_common/src/thread_runner.h"
#include "ncode_common/src/viz/web_page.h"
#include "common.h"
#include "geo/geo.h"
#include "metrics/metrics.h"
#include "opt/ctr.h"
#include "opt/opt.h"
#include "opt/path_provider.h"

DEFINE_string(cities_file, "cities5000.txt", "Location of the cities file");
DEFINE_string(topology_file, "", "Topology file");
DEFINE_string(tm_file, "", "Traffic matrix file");
DEFINE_double(link_capacity_scale, 1.0, "By how much to scale all links");
DEFINE_double(delay_scale, 1.0, "By how much to scale the delays of all links");
DEFINE_double(
    light_speed, 200.0,
    "Speed of light (in km per millisecond), default is speed in fiber");
DEFINE_double(link_speed_Mbps, 1000, "All new links will be this fast");
DEFINE_double(scale_step, 1.01, "By how much to scale the TM each step");
DEFINE_double(capacity_grow_threshold, 1.3,
              "What the limit is for growing capacity");
DEFINE_string(
    optimizer, "CTR",
    "The optimizer to use. One of SP,CTR,MinMax,MinMaxLD,MinMaxK10,B4.");
DEFINE_uint64(threads, 4, "Number of threads to use");

using namespace std::chrono;

// Elements of a traffic matrix.
using TMElements = std::vector<nc::lp::DemandMatrixElement>;

// Number of flows in the most loaded aggregate.
static constexpr size_t kTopAggregateFlowCount = 10000;

static auto* total_delay_ms =
    nc::metrics::DefaultMetricManager()
        -> GetThreadSafeMetric<uint32_t>("total_delay_ms", "Sum of all delays");

static auto* max_link_utilization =
    nc::metrics::DefaultMetricManager() -> GetThreadSafeMetric<double>(
        "max_link_utilization", "Maximum link utilization");

static auto* capacity_added =
    nc::metrics::DefaultMetricManager() -> GetThreadSafeMetric<double>(
        "capacity_added", "Capacity added to the network (in Mbps)");

namespace ctr {

static std::unique_ptr<TrafficMatrix> FromDemandMatrix(
    const nc::lp::DemandMatrix& demand_matrix) {
  nc::net::Bandwidth max_demand = nc::net::Bandwidth::Zero();
  for (const auto& element : demand_matrix.elements()) {
    max_demand = std::max(max_demand, element.demand);
  }

  // Will assign flow counts so that the max bandwidth aggregate has
  // kTopAggregateFlowCount, and all other aggregates proportionally less.
  std::map<AggregateId, DemandAndFlowCount> demands_and_counts;
  for (const auto& element : demand_matrix.elements()) {
    size_t flow_count = kTopAggregateFlowCount * (element.demand / max_demand);
    flow_count = std::max(1ul, flow_count);

    AggregateId id(element.src, element.dst);
    demands_and_counts[id] = {element.demand, flow_count};
  }

  return nc::make_unique<TrafficMatrix>(demand_matrix.graph(),
                                        demands_and_counts);
}

static std::unique_ptr<RoutingConfiguration> RunOptimizer(
    const nc::lp::DemandMatrix& demand_matrix, PathProvider* provider) {
  auto tm = FromDemandMatrix(demand_matrix);

  std::unique_ptr<Optimizer> opt;
  if (FLAGS_optimizer == "CTR") {
    opt = nc::make_unique<CTROptimizer>(provider, 1.0, false, false);
  } else if (FLAGS_optimizer == "MinMax") {
    opt = nc::make_unique<MinMaxOptimizer>(provider, 1.0, false);
  } else if (FLAGS_optimizer == "MinMaxLD") {
    opt = nc::make_unique<MinMaxOptimizer>(provider, 1.0, true);
  } else if (FLAGS_optimizer == "MinMaxK10") {
    opt = nc::make_unique<MinMaxPathBasedOptimizer>(provider, 1.0, true, 10);
  } else if (FLAGS_optimizer == "B4") {
    opt = nc::make_unique<B4Optimizer>(provider, false, 1.0);
  } else if (FLAGS_optimizer == "SP") {
    opt = nc::make_unique<ShortestPathOptimizer>(provider);
  } else {
    LOG(FATAL) << "Bad optimizer " << FLAGS_optimizer;
  }

  return opt->Optimize(*tm);
}

// Scores a combination of topology and TM.
static std::pair<double, bool> Cost(const nc::net::GraphBuilder& topology,
                                    const TMElements& tm_elements) {
  nc::net::GraphStorage graph(topology);
  PathProvider path_provider(&graph);
  nc::lp::DemandMatrix demand_matrix(tm_elements, &graph);
  auto routing = RunOptimizer(demand_matrix, &path_provider);

  double total_oversubscription = 0;
  double max_utilization = 0;
  nc::net::GraphLinkMap<double> link_utilizations = routing->LinkUtilizations();
  for (const auto& link_and_utilization : link_utilizations) {
    double utilization = *(link_and_utilization.second);

    double oversubscription = utilization - 1.0;
    oversubscription = std::max(0.0, oversubscription);
    total_oversubscription += oversubscription;

    max_utilization = std::max(max_utilization, utilization);
  }

  return {total_oversubscription, max_utilization <= 1.0};
}

// Records information about the routing.
static void RecordRoutingConfig(const RoutingConfiguration& routing,
                                uint32_t step) {
  auto* total_path_handle = total_delay_ms->GetHandle();
  auto* link_utilization_handle = max_link_utilization->GetHandle();

  nc::net::Delay total_delay = routing.TotalPerFlowDelay();
  total_path_handle->AddValue(duration_cast<milliseconds>(total_delay).count());

  double max_utilization = routing.MaxLinkUtilization();
  link_utilization_handle->AddValue(max_utilization);

  nc::viz::HtmlPage page;
  routing.ToHTML(&page);
  nc::File::WriteStringToFile(page.Construct(),
                              nc::StrCat("top_grow_step_", step, ".html"));
}

// Will add a single link (or increase the capacity of existing one) that
// minimizes the overall cost.
static nc::net::GraphBuilder GrowNetwork(
    const nc::net::GraphBuilder& topology, const TMElements& tm_elements,
    const std::map<std::pair<std::string, std::string>, milliseconds>& delays) {
  nc::net::GraphBuilder to_return = topology;
  double total_added = 0.0;
  while (true) {
    nc::net::GraphBuilder best_topology;
    double best_cost = std::numeric_limits<double>::max();
    bool best_fits = false;
    std::string best_link_to_string;

    std::vector<nc::net::GraphBuilder> to_run;
    std::vector<std::string> links_added_to_string;
    std::set<std::string> all_nodes = topology.AllNodeNames();
    for (const std::string& src : all_nodes) {
      for (const std::string& dst : all_nodes) {
        if (src >= dst) {
          continue;
        }

        nc::net::GraphBuilder new_topology = to_return;
        nc::net::Delay delay =
            nc::FindOrDieNoPrint(delays, std::make_pair(src, dst));
        nc::net::Bandwidth bw =
            nc::net::Bandwidth::FromMBitsPerSecond(FLAGS_link_speed_Mbps);
        new_topology.AddLink({src, dst, bw, delay});
        new_topology.AddLink({dst, src, bw, delay});
        new_topology.RemoveMultipleLinks();
        to_run.emplace_back(new_topology);
        links_added_to_string.emplace_back(nc::StrCat(src, "->", dst));
      }
    }

    std::vector<std::unique_ptr<std::pair<double, bool>>> output =
        nc::RunInParallelWithResult<nc::net::GraphBuilder,
                                    std::pair<double, bool>>(
            to_run, [&tm_elements](const nc::net::GraphBuilder& builder) {
              return nc::make_unique<std::pair<double, bool>>(
                  Cost(builder, tm_elements));
            }, FLAGS_threads);

    for (size_t i = 0; i < to_run.size(); ++i) {
      const nc::net::GraphBuilder& new_topology = to_run[i];
      double cost;
      bool fits;
      std::tie(cost, fits) = *(output[i]);

      if (cost < best_cost) {
        best_topology = new_topology;
        best_cost = cost;
        best_link_to_string = links_added_to_string[i];
        best_fits = fits;
      }
    }

    CHECK(best_cost != std::numeric_limits<double>::max());
    LOG(INFO) << "Growing network, best cost " << best_cost << " will add "
              << best_link_to_string << " fits " << best_fits;
    total_added += FLAGS_link_speed_Mbps;

    to_return = best_topology;
    if (best_fits) {
      break;
    }
  }

  capacity_added->GetHandle()->AddValue(total_added);
  return to_return;
}

static void Run(
    const nc::net::GraphBuilder& topology, const TMElements& tm_elements,
    const std::map<std::pair<std::string, std::string>, milliseconds>& delays) {
  nc::net::GraphBuilder current_topology = topology;
  TMElements current_elements = tm_elements;

  for (size_t i = 0; i < 100; ++i) {
    LOG(INFO) << "Step " << i;

    // Will first run to record information.
    nc::net::GraphStorage graph(current_topology);
    PathProvider path_provider(&graph);

    nc::lp::DemandMatrix demand_matrix(current_elements, &graph);
    auto output = RunOptimizer(demand_matrix, &path_provider);
    RecordRoutingConfig(*output, i);

    // Will grow the TM to figure out if any new links need to be added.
    auto scaled_demands = demand_matrix.Scale(FLAGS_capacity_grow_threshold);
    output = RunOptimizer(*scaled_demands, &path_provider);

    bool all_fit = output->MaxLinkUtilization() <= 1.0;
    if (!all_fit) {
      // Need to grow the network.
      current_topology =
          GrowNetwork(topology, scaled_demands->elements(), delays);
    } else {
      capacity_added->GetHandle()->AddValue(0.0);
    }

    // Perform a single step.
    scaled_demands = demand_matrix.Scale(FLAGS_scale_step);
    current_elements = scaled_demands->elements();
  }
}

}  // namespace ctr

static const nc::geo::CityData* GetCity(const std::string& name,
                                        nc::geo::Localizer* localizer) {
  nc::geo::FindCityRequest request;
  request.ascii_name = name;

  const nc::geo::CityData* city_data = localizer->FindCityOrNull(request);
  LOG(INFO) << name << " localized to " << city_data->ToString();
  CHECK(city_data != nullptr);
  return city_data;
}

int main(int argc, char** argv) {
  gflags::ParseCommandLineFlags(&argc, &argv, true);
  nc::metrics::InitMetrics();
  nc::geo::Localizer localizer(FLAGS_cities_file);

  std::vector<std::string> node_order;
  nc::net::GraphBuilder builder = nc::net::LoadRepetitaOrDie(
      nc::File::ReadFileToStringOrDie(FLAGS_topology_file), &node_order);
  builder.RemoveMultipleLinks();
  builder.ScaleCapacity(FLAGS_link_capacity_scale);
  builder.ScaleDelay(FLAGS_delay_scale);

  nc::net::GraphStorage graph(builder);
  std::unique_ptr<nc::lp::DemandMatrix> demand_matrix =
      nc::lp::DemandMatrix::LoadRepetitaOrDie(
          nc::File::ReadFileToStringOrDie(FLAGS_tm_file), node_order, &graph);

  // Will record the delays between all N * (N - 1) nodes.
  std::map<std::pair<std::string, std::string>, milliseconds> delays;
  nc::net::GraphNodeSet all_nodes = graph.AllNodes();
  for (nc::net::GraphNodeIndex src : all_nodes) {
    for (nc::net::GraphNodeIndex dst : all_nodes) {
      if (src == dst) {
        continue;
      }

      const std::string& src_id = graph.GetNode(src)->id();
      const std::string& dst_id = graph.GetNode(dst)->id();

      const nc::geo::CityData* src_city = GetCity(src_id, &localizer);
      const nc::geo::CityData* dst_city = GetCity(dst_id, &localizer);
      double distance_km = src_city->DistanceKm(*dst_city);

      uint32_t delay_ms = distance_km / FLAGS_light_speed;
      delay_ms = std::max(delay_ms, static_cast<uint32_t>(1));
      delays[{src_id, dst_id}] = milliseconds(delay_ms);
    }
  }

  ctr::Run(builder, demand_matrix->elements(), delays);
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Plenluno All rights reserved.

#include <libnode/fs.h>
#include <libnode/path.h>

#include "./gtest_common.h"

namespace libj {
namespace node {

class FsCallback : LIBJ_JS_FUNCTION(FsCallback)
 public:
    FsCallback() : args_(JsArray::null()) {}

    JsArray::CPtr getArgs() { return args_; }

    virtual Value operator()(JsArray::Ptr args) {
        args_ = args;
        return Status::OK;
    }

 private:
    JsArray::Ptr args_;
};

TEST(GTestFs, TestWriteFile) {
    fs::writeFile(
        String::null(), Buffer::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::writeFile(str("hello.txt"), Buffer::create(str("hello")), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
}

TEST(GTestFs, TestStat) {
    fs::stat(String::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::stat(String::null(), cb);
    node::run();
    ASSERT_TRUE(!!cb->getArgs()->getCPtr<Error>(0));

    fs::stat(str("hello.txt"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));

    fs::Stats::Ptr stats = cb->getArgs()->getPtr<fs::Stats>(1);
    ASSERT_TRUE(!!stats);
    ASSERT_TRUE(stats->isFile());
    ASSERT_FALSE(stats->isDirectory());
    ASSERT_EQ(5, to<Size>(stats->get(fs::STAT_SIZE)));
}

TEST(GTestFs, TestOpen) {
    fs::open(String::null(), fs::W, JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::open(String::null(), fs::W, cb);
    node::run();
    ASSERT_TRUE(!!cb->getArgs()->getCPtr<Error>(0));
}

TEST(GTestFs, TestClose) {
    fs::close(Object::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::close(Object::null(), cb);
    node::run();
    ASSERT_TRUE(!!cb->getArgs()->getCPtr<Error>(0));
}

TEST(GTestFs, TestRead) {
    fs::read(
        Object::null(),
        Buffer::null(),
        0, 0, 0,
        JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::read(
        Object::null(),
        Buffer::create(),
        1, 1, 1, cb);
    node::run();
    ASSERT_TRUE(!!cb->getArgs()->getCPtr<Error>(0));
}

TEST(GTestFs, TestReadFile) {
    fs::readFile(String::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::readFile(str("hello.txt"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
    Buffer::CPtr buf = cb->getArgs()->getCPtr<Buffer>(1);
    ASSERT_TRUE(!!buf);
    ASSERT_TRUE(buf->toString()->equals(str("hello")));
}

TEST(GTestFs, TestAppendFile) {
    fs::appendFile(
        String::null(), Buffer::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::appendFile(str("hello.txt"), Buffer::create(str(" world")), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));

    fs::readFile(str("hello.txt"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
    Buffer::CPtr buf = cb->getArgs()->getCPtr<Buffer>(1);
    ASSERT_TRUE(!!buf);
    ASSERT_TRUE(buf->toString()->equals(str("hello world")));
}

TEST(GTestFs, TestReaddir) {
    fs::readdir(String::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::readdir(str("."), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
    JsArray::CPtr a = cb->getArgs()->getCPtr<JsArray>(1);
    ASSERT_TRUE(a->contains(str("hello.txt")));
}

TEST(GTestFs, TestRename) {
    fs::rename(
        String::null(), String::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::rename(str("hello.txt"), str("hell.txt"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
}

TEST(GTestFs, TestTruncate) {
    fs::truncate(String::null(), 0, JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::truncate(str("hell.txt"), 4, cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));

    fs::readFile(str("hell.txt"), cb);
    node::run();
    Buffer::CPtr buf = cb->getArgs()->getCPtr<Buffer>(1);
    ASSERT_TRUE(!!buf);
    ASSERT_EQ(4, buf->length());
}

TEST(GTestFs, TestMkdir) {
    fs::mkdir(String::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::mkdir(str("mydir"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
}

TEST(GTestFs, TestSymlink) {
    fs::symlink(
        String::null(), String::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::symlink(str("mydir"), str("mylink"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
}

TEST(GTestFs, TestLstat) {
    FsCallback::Ptr cb(new FsCallback());
    fs::lstat(str("mydir"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));

    fs::Stats::Ptr stats = cb->getArgs()->getPtr<fs::Stats>(1);
    ASSERT_TRUE(!!stats);
    ASSERT_TRUE(stats->isDirectory());
    ASSERT_FALSE(stats->isSocket());

    fs::lstat(str("mylink"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));

    stats = cb->getArgs()->getPtr<fs::Stats>(1);
    ASSERT_TRUE(!!stats);
    ASSERT_TRUE(stats->isSymbolicLink());
    ASSERT_FALSE(stats->isBlockDevice());
}

TEST(GTestFs, TestRealpath) {
    fs::realpath(String::null(), JsFunction::null());  // no crash
    node::run();

    String::CPtr current = path::resolve(JsArray::null());

    FsCallback::Ptr cb(new FsCallback());
    fs::realpath(String::null(), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
    ASSERT_TRUE(cb->getArgs()->getCPtr<String>(1)->equals(current));

    fs::realpath(str("./mylink"), cb);
    String::CPtr real = current->concat(str("/mydir"));
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
    ASSERT_TRUE(cb->getArgs()->getCPtr<String>(1)->equals(real));
}

TEST(GTestFs, TestRmdir) {
    fs::rmdir(String::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::rmdir(str("mydir"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
}

TEST(GTestFs, TestUnlink) {
    fs::unlink(String::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::unlink(str("hell.txt"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));

    fs::unlink(str("mylink"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
}

}  // namespace node
}  // namespace libj
<commit_msg>fix a wrong windows path<commit_after>// Copyright (c) 2013 Plenluno All rights reserved.

#include <libnode/fs.h>
#include <libnode/path.h>

#include "./gtest_common.h"

namespace libj {
namespace node {

class FsCallback : LIBJ_JS_FUNCTION(FsCallback)
 public:
    FsCallback() : args_(JsArray::null()) {}

    JsArray::CPtr getArgs() { return args_; }

    virtual Value operator()(JsArray::Ptr args) {
        args_ = args;
        return Status::OK;
    }

 private:
    JsArray::Ptr args_;
};

TEST(GTestFs, TestWriteFile) {
    fs::writeFile(
        String::null(), Buffer::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::writeFile(str("hello.txt"), Buffer::create(str("hello")), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
}

TEST(GTestFs, TestStat) {
    fs::stat(String::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::stat(String::null(), cb);
    node::run();
    ASSERT_TRUE(!!cb->getArgs()->getCPtr<Error>(0));

    fs::stat(str("hello.txt"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));

    fs::Stats::Ptr stats = cb->getArgs()->getPtr<fs::Stats>(1);
    ASSERT_TRUE(!!stats);
    ASSERT_TRUE(stats->isFile());
    ASSERT_FALSE(stats->isDirectory());
    ASSERT_EQ(5, to<Size>(stats->get(fs::STAT_SIZE)));
}

TEST(GTestFs, TestOpen) {
    fs::open(String::null(), fs::W, JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::open(String::null(), fs::W, cb);
    node::run();
    ASSERT_TRUE(!!cb->getArgs()->getCPtr<Error>(0));
}

TEST(GTestFs, TestClose) {
    fs::close(Object::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::close(Object::null(), cb);
    node::run();
    ASSERT_TRUE(!!cb->getArgs()->getCPtr<Error>(0));
}

TEST(GTestFs, TestRead) {
    fs::read(
        Object::null(),
        Buffer::null(),
        0, 0, 0,
        JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::read(
        Object::null(),
        Buffer::create(),
        1, 1, 1, cb);
    node::run();
    ASSERT_TRUE(!!cb->getArgs()->getCPtr<Error>(0));
}

TEST(GTestFs, TestReadFile) {
    fs::readFile(String::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::readFile(str("hello.txt"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
    Buffer::CPtr buf = cb->getArgs()->getCPtr<Buffer>(1);
    ASSERT_TRUE(!!buf);
    ASSERT_TRUE(buf->toString()->equals(str("hello")));
}

TEST(GTestFs, TestAppendFile) {
    fs::appendFile(
        String::null(), Buffer::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::appendFile(str("hello.txt"), Buffer::create(str(" world")), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));

    fs::readFile(str("hello.txt"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
    Buffer::CPtr buf = cb->getArgs()->getCPtr<Buffer>(1);
    ASSERT_TRUE(!!buf);
    ASSERT_TRUE(buf->toString()->equals(str("hello world")));
}

TEST(GTestFs, TestReaddir) {
    fs::readdir(String::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::readdir(str("."), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
    JsArray::CPtr a = cb->getArgs()->getCPtr<JsArray>(1);
    ASSERT_TRUE(a->contains(str("hello.txt")));
}

TEST(GTestFs, TestRename) {
    fs::rename(
        String::null(), String::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::rename(str("hello.txt"), str("hell.txt"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
}

TEST(GTestFs, TestTruncate) {
    fs::truncate(String::null(), 0, JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::truncate(str("hell.txt"), 4, cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));

    fs::readFile(str("hell.txt"), cb);
    node::run();
    Buffer::CPtr buf = cb->getArgs()->getCPtr<Buffer>(1);
    ASSERT_TRUE(!!buf);
    ASSERT_EQ(4, buf->length());
}

TEST(GTestFs, TestMkdir) {
    fs::mkdir(String::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::mkdir(str("mydir"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
}

TEST(GTestFs, TestSymlink) {
    fs::symlink(
        String::null(), String::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::symlink(str("mydir"), str("mylink"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
}

TEST(GTestFs, TestLstat) {
    FsCallback::Ptr cb(new FsCallback());
    fs::lstat(str("mydir"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));

    fs::Stats::Ptr stats = cb->getArgs()->getPtr<fs::Stats>(1);
    ASSERT_TRUE(!!stats);
    ASSERT_TRUE(stats->isDirectory());
    ASSERT_FALSE(stats->isSocket());

    fs::lstat(str("mylink"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));

    stats = cb->getArgs()->getPtr<fs::Stats>(1);
    ASSERT_TRUE(!!stats);
    ASSERT_TRUE(stats->isSymbolicLink());
    ASSERT_FALSE(stats->isBlockDevice());
}

TEST(GTestFs, TestRealpath) {
    fs::realpath(String::null(), JsFunction::null());  // no crash
    node::run();

    String::CPtr current = path::resolve(JsArray::null());

    FsCallback::Ptr cb(new FsCallback());
    fs::realpath(String::null(), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
    ASSERT_TRUE(cb->getArgs()->getCPtr<String>(1)->equals(current));

    fs::realpath(str("./mylink"), cb);
    JsArray::Ptr paths = JsArray::create();
    paths->add(current);
    paths->add(str("./mydir"));
    String::CPtr real = path::resolve(paths);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
    ASSERT_TRUE(cb->getArgs()->getCPtr<String>(1)->equals(real));
}

TEST(GTestFs, TestRmdir) {
    fs::rmdir(String::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::rmdir(str("mydir"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
}

TEST(GTestFs, TestUnlink) {
    fs::unlink(String::null(), JsFunction::null());  // no crash
    node::run();

    FsCallback::Ptr cb(new FsCallback());
    fs::unlink(str("hell.txt"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));

    fs::unlink(str("mylink"), cb);
    node::run();
    ASSERT_TRUE(!cb->getArgs()->getCPtr<Error>(0));
}

}  // namespace node
}  // namespace libj
<|endoftext|>