<commit_before>#ifndef SILICIUM_REACTIVE_COROUTINE_HPP
#define SILICIUM_REACTIVE_COROUTINE_HPP

#include <silicium/exchange.hpp>
#include <silicium/yield_context.hpp>
#include <silicium/fast_variant.hpp>
#include <boost/coroutine/all.hpp>

namespace Si
{
	namespace detail
	{
		template <class Element>
		struct result
		{
			Element value;

			result()
			{
			}

			explicit result(Element value)
				: value(std::move(value))
			{
			}

#ifdef _MSC_VER
			result(result &&other)
				: value(std::move(other.value))
			{
			}

			result &operator = (result &&other)
			{
				value = std::move(other.value);
				return *this;
			}

			result(result const &other)
				: value(other.value)
			{
			}

			result &operator = (result const &other)
			{
				value = other.value;
				return *this;
			}
#endif
		};

		struct yield
		{
			Si::observable<nothing> *target;
		};

		template <class Element>
		struct make_command
		{
			typedef Si::fast_variant<result<Element>, yield> type;
		};

		template <class Element>
		struct coroutine_yield_context_impl : detail::yield_context_impl<Element>
		{
			typedef typename boost::coroutines::coroutine<typename detail::make_command<Element>::type>::push_type consumer_type;

			explicit coroutine_yield_context_impl(consumer_type &consumer)
				: consumer(&consumer)
			{
			}

			virtual void push_result(Element result) SILICIUM_OVERRIDE
			{
				(*consumer)(detail::result<Element>(std::move(result)));
			}

			virtual void get_one(observable<nothing> &target) SILICIUM_OVERRIDE
			{
				(*consumer)(detail::yield{&target});
			}

		private:

			consumer_type *consumer = nullptr;
		};
	}

	template <class Element>
	struct coroutine_observable
		: private Si::observer<nothing>
		, public boost::static_visitor<> //TODO make private
	{
		typedef Element element_type;

		coroutine_observable()
		{
		}

		coroutine_observable(coroutine_observable &&other)
		{
			*this = std::move(other);
		}

		coroutine_observable &operator = (coroutine_observable &&other)
		{
			coro_ = std::move(other.coro_);
			action = std::move(other.action);
			receiver_ = std::move(other.receiver_);
			return *this;
		}

		template <class Action>
		explicit coroutine_observable(Action &&action)
			: action(std::forward<Action>(action))
		{
		}

		void async_get_one(Si::observer<element_type> &receiver)
		{
			receiver_ = &receiver;
			next();
		}

		//TODO make private
		void operator()(detail::result<element_type> command)
		{
			return Si::exchange(receiver_, nullptr)->got_element(std::move(command.value));
		}

		//TODO make private
		void operator()(detail::yield command)
		{
			command.target->async_get_one(*this);
		}

	private:

		typedef typename detail::make_command<element_type>::type command_type;
		typedef typename boost::coroutines::coroutine<command_type>::pull_type coroutine_type;
		using coroutine_holder =
#ifdef _MSC_VER
			std::shared_ptr<coroutine_type>
#else
			coroutine_type
#endif
			;

		coroutine_holder coro_;
		std::function<void (yield_context<Element> &)> action;
		Si::observer<Element> *receiver_ = nullptr;

		coroutine_type &coro()
		{
			return
#ifdef _MSC_VER
				*
#endif
				coro_;
		}

		virtual void got_element(nothing) SILICIUM_OVERRIDE
		{
			next();
		}

		virtual void ended() SILICIUM_OVERRIDE
		{
			Si::exchange(receiver_, nullptr)->ended();
		}

		void next()
		{
			if (action)
			{
				auto bound_action = action;
				coro_ =
#ifdef _MSC_VER
					std::make_shared<coroutine_type>
#else
					coroutine_type
#endif
						([bound_action](typename boost::coroutines::coroutine<command_type>::push_type &push)
						{
							detail::coroutine_yield_context_impl<Element> yield_impl(push);
							yield_context<Element> yield(yield_impl); //TODO: save this indirection
							return bound_action(yield);
						});
				action = nullptr;
			}
			else if (coro())
			{
				coro()();
			}
			if (coro())
			{
				command_type command = coro().get();
				return Si::apply_visitor(*this, command);
			}
			else
			{
				Si::exchange(receiver_, nullptr)->ended();
			}
		}
	};

	template <class Element, class Action>
	auto make_coroutine(Action &&action) -> coroutine_observable<Element>
	{
		return coroutine_observable<Element>(std::forward<Action>(action));
	}
}

#endif
<commit_msg>fix mistake in coroutine when the observable given to get_one ended<commit_after>#ifndef SILICIUM_REACTIVE_COROUTINE_HPP
#define SILICIUM_REACTIVE_COROUTINE_HPP

#include <silicium/exchange.hpp>
#include <silicium/yield_context.hpp>
#include <silicium/fast_variant.hpp>
#include <boost/coroutine/all.hpp>

namespace Si
{
	namespace detail
	{
		template <class Element>
		struct result
		{
			Element value;

			result()
			{
			}

			explicit result(Element value)
				: value(std::move(value))
			{
			}

#ifdef _MSC_VER
			result(result &&other)
				: value(std::move(other.value))
			{
			}

			result &operator = (result &&other)
			{
				value = std::move(other.value);
				return *this;
			}

			result(result const &other)
				: value(other.value)
			{
			}

			result &operator = (result const &other)
			{
				value = other.value;
				return *this;
			}
#endif
		};

		struct yield
		{
			Si::observable<nothing> *target;
		};

		template <class Element>
		struct make_command
		{
			typedef Si::fast_variant<result<Element>, yield> type;
		};

		template <class Element>
		struct coroutine_yield_context_impl : detail::yield_context_impl<Element>
		{
			typedef typename boost::coroutines::coroutine<typename detail::make_command<Element>::type>::push_type consumer_type;

			explicit coroutine_yield_context_impl(consumer_type &consumer)
				: consumer(&consumer)
			{
			}

			virtual void push_result(Element result) SILICIUM_OVERRIDE
			{
				(*consumer)(detail::result<Element>(std::move(result)));
			}

			virtual void get_one(observable<nothing> &target) SILICIUM_OVERRIDE
			{
				(*consumer)(detail::yield{&target});
			}

		private:

			consumer_type *consumer = nullptr;
		};
	}

	template <class Element>
	struct coroutine_observable
		: private Si::observer<nothing>
		, public boost::static_visitor<> //TODO make private
	{
		typedef Element element_type;

		coroutine_observable()
		{
		}

		coroutine_observable(coroutine_observable &&other)
		{
			*this = std::move(other);
		}

		coroutine_observable &operator = (coroutine_observable &&other)
		{
			coro_ = std::move(other.coro_);
			action = std::move(other.action);
			receiver_ = std::move(other.receiver_);
			return *this;
		}

		template <class Action>
		explicit coroutine_observable(Action &&action)
			: action(std::forward<Action>(action))
		{
		}

		void async_get_one(Si::observer<element_type> &receiver)
		{
			receiver_ = &receiver;
			next();
		}

		//TODO make private
		void operator()(detail::result<element_type> command)
		{
			return Si::exchange(receiver_, nullptr)->got_element(std::move(command.value));
		}

		//TODO make private
		void operator()(detail::yield command)
		{
			command.target->async_get_one(*this);
		}

	private:

		typedef typename detail::make_command<element_type>::type command_type;
		typedef typename boost::coroutines::coroutine<command_type>::pull_type coroutine_type;
		using coroutine_holder =
#ifdef _MSC_VER
			std::shared_ptr<coroutine_type>
#else
			coroutine_type
#endif
			;

		coroutine_holder coro_;
		std::function<void (yield_context<Element> &)> action;
		Si::observer<Element> *receiver_ = nullptr;

		coroutine_type &coro()
		{
			return
#ifdef _MSC_VER
				*
#endif
				coro_;
		}

		virtual void got_element(nothing) SILICIUM_OVERRIDE
		{
			next();
		}

		virtual void ended() SILICIUM_OVERRIDE
		{
			next();
		}

		void next()
		{
			if (action)
			{
				auto bound_action = action;
				coro_ =
#ifdef _MSC_VER
					std::make_shared<coroutine_type>
#else
					coroutine_type
#endif
						([bound_action](typename boost::coroutines::coroutine<command_type>::push_type &push)
						{
							detail::coroutine_yield_context_impl<Element> yield_impl(push);
							yield_context<Element> yield(yield_impl); //TODO: save this indirection
							return bound_action(yield);
						});
				action = nullptr;
			}
			else if (coro())
			{
				coro()();
			}
			if (coro())
			{
				command_type command = coro().get();
				return Si::apply_visitor(*this, command);
			}
			else
			{
				Si::exchange(receiver_, nullptr)->ended();
			}
		}
	};

	template <class Element, class Action>
	auto make_coroutine(Action &&action) -> coroutine_observable<Element>
	{
		return coroutine_observable<Element>(std::forward<Action>(action));
	}
}

#endif
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2016 DeepCortex GmbH <legal@eventql.io>
 * Authors:
 *   - Laura Schlimmer <laura@eventql.io>
 *
 * This program is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License ("the license") as
 * published by the Free Software Foundation, either version 3 of the License,
 * or any later version.
 *
 * In accordance with Section 7(e) of the license, the licensing of the Program
 * under the license does not imply a trademark license. Therefore any rights,
 * title and interest in our trademarks remain entirely with us.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the license for more details.
 *
 * You can be released from the requirements of the license by purchasing a
 * commercial license. Buying such a license is mandatory as soon as you develop
 * commercial activities involving this program without disclosing the source
 * code of your own applications
 */
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include <poll.h>
#include <thread>
#include "eventql/eventql.h"
#include "eventql/util/application.h"
#include "eventql/util/logging.h"
#include "eventql/util/UnixTime.h"
#include "eventql/util/return_code.h"
#include "eventql/util/cli/CLI.h"
#include "eventql/util/io/inputstream.h"
#include "eventql/util/cli/term.h"
#include "eventql/util/cli/flagparser.h"
#include "eventql/util/thread/threadpool.h"
#include "eventql/cli/cli_config.h"

ReturnCode sendQuery(
    const String& query,
    const String& qry_db,
    evql_client_t* client) {
  uint64_t qry_flags = 0;
  if (!qry_db.empty()) {
    qry_flags |= EVQL_QUERY_SWITCHDB;
  }

  int rc = evql_query(client, query.c_str(), qry_db.c_str(), qry_flags);

  size_t result_ncols;
  if (rc == 0) {
    rc = evql_num_columns(client, &result_ncols);
  }

  while (rc >= 0) {
    const char** fields;
    size_t* field_lens;
    rc = evql_fetch_row(client, &fields, &field_lens);
    if (rc < 1) {
      break;
    }
  }

  evql_client_releasebuffers(client);

  if (rc == -1) {
    return ReturnCode::error("EIOERROR", evql_client_geterror(client));
  } else {
    return ReturnCode::success();
  }

}

void print(
    size_t num_errors,
    size_t num_succes,
    UnixTime start,
    OutputStream* stdout_os) {
  UnixTime now;
  auto duration = now - start;
  stdout_os->write(StringUtil::format(
    "total   successful    error      milliseconds\n"
    "   $0      $1            $2      $3\n\n",
    num_errors + num_succes,
    num_succes,
    num_errors,
    duration.milliseconds()
  ));
}

static constexpr auto kMaxErrors = 10;

struct TimeWindow {
  static constexpr auto kMillisPerWindow = 1 * kMillisPerSecond;

  void clear() {
    num_requests = 0;
    UnixTime now;
    start = now;
  }

  int64_t getRemainingMillis() {
    UnixTime now;
    auto duration = now - start;
    return kMillisPerWindow - duration.milliseconds();
  }

  size_t num_requests;
  UnixTime start;
};

int main(int argc, const char** argv) {
  cli::FlagParser flags;

  flags.defineFlag(
      "help",
      cli::FlagParser::T_SWITCH,
      false,
      "?",
      NULL,
      "help",
      "<help>");

  flags.defineFlag(
      "version",
      cli::FlagParser::T_SWITCH,
      false,
      "v",
      NULL,
      "print version",
      "<switch>");

  flags.defineFlag(
      "host",
      cli::FlagParser::T_STRING,
      false,
      "h",
      NULL,
      "eventql server hostname",
      "<host>");

  flags.defineFlag(
      "port",
      cli::FlagParser::T_INTEGER,
      false,
      "p",
      NULL,
      "eventql server port",
      "<port>");

  flags.defineFlag(
      "database",
      cli::FlagParser::T_STRING,
      false,
      "d",
      NULL,
      "database",
      "<db>");

  flags.defineFlag(
      "query",
      cli::FlagParser::T_STRING,
      true,
      "q",
      NULL,
      "query str",
      "<query>");

  flags.defineFlag(
      "threads",
      cli::FlagParser::T_INTEGER,
      false,
      "t",
      "10",
      "number of threads",
      "<threads>");

  flags.defineFlag(
      "rate",
      cli::FlagParser::T_INTEGER,
      false,
      "r",
      "1",
      "number of requests per second",
      "<rate>");

  flags.defineFlag(
      "max",
      cli::FlagParser::T_INTEGER,
      false,
      "n",
      NULL,
      "number of requests per second",
      "<rate>");

  flags.defineFlag(
      "loglevel",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);
  Vector<String> cmd_argv = flags.getArgv();
  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  Application::init();
  Application::logToStderr("evqlbenchmark");
  auto stdin_is = InputStream::getStdin();
  auto stdout_os = OutputStream::getStdout();
  auto stderr_os = OutputStream::getStderr();

  bool print_help = flags.isSet("help");
  bool print_version = flags.isSet("version");
  if (print_version || print_help) {
    auto stdout_os = OutputStream::getStdout();
    stdout_os->write(
        StringUtil::format(
            "EventQL $0 ($1)\n"
            "Copyright (c) 2016, DeepCortex GmbH. All rights reserved.\n\n",
            eventql::kVersionString,
            eventql::kBuildID));
  }

  if (print_version) {
    return 1;
  }

  if (print_help) {
    stdout_os->write(
      "Usage: $ evqlbenchmark [OPTIONS] <command> [<args>]\n\n"
      "   -D, --database <db>       Select a database\n"
      "   -h, --host <hostname>     Set the EventQL server hostname\n"
      "   -p, --port <port>         Set the EventQL server port\n"
      "   -?, --help <topic>        Display a command's help text and exit\n"
      "   -v, --version             Display the version of this binary and exit\n\n"
      "evqlbenchmark commands:\n"
      );
    return 1;
  }

  logInfo("evqlbenchmark", "starting benchmark");
  /* options */
  eventql::ProcessConfigBuilder cfg_builder;
  {
    auto status = cfg_builder.loadDefaultConfigFile("evql");
    if (!status.isSuccess()) {
      logFatal("evqlbenchmark", "error while loading config file %s", status.message());
      return 0;
    }
  }

  if (flags.isSet("host")) {
    cfg_builder.setProperty("evql", "host", flags.getString("host"));
  }

  if (flags.isSet("port")) {
    cfg_builder.setProperty(
        "evql",
        "port",
        StringUtil::toString(flags.getInt("port")));
  }

  if (flags.isSet("database")) {
    cfg_builder.setProperty("evql", "database", flags.getString("database"));
  }

  eventql::cli::CLIConfig cfg(cfg_builder.getConfig());

  String qry_db;
  if (flags.isSet("database")) {
    qry_db = flags.getString("database");
  }

  auto qry_str = flags.getString("query");
  auto rate = flags.getInt("rate");
  auto num_threads = flags.getInt("threads");
  size_t max_requests;
  bool has_max = false;
  if (flags.isSet("max")) {
    max_requests = flags.getInt("max");
    has_max = true;
  }

  const UnixTime global_start;

  std::mutex m;
  TimeWindow twindow;
  auto errors = 0;
  auto requests_sent = 0;

  Vector<std::thread> threads;
  for (size_t i = 0; i < num_threads; ++i) {
    threads.emplace_back(std::thread([&] () {

      /* connect to eventql client */
      auto client = evql_client_init();
      if (!client) {
        logFatal("evqlbenchmark", "can't initialize eventql client");
        return 0;
      }

      {
        auto rc = evql_client_connect(
            client,
            cfg.getHost().c_str(),
            cfg.getPort(),
            0);

        if (rc < 0) {
          logFatal("evqlbenchmark", "can't connect to eventql client: $0", evql_client_geterror(client));
          return 0;
        }
      }

      for (;;) {
        /* check remaining time in current timewindow */
        m.lock();
        /* start a new timewindow */
        if (twindow.getRemainingMillis() <= 0) {
          twindow.clear();
          m.unlock();
          continue;
        }
        m.unlock();

        /* send query */
        auto rc = sendQuery(qry_str, qry_db, client);
        m.lock();
        if (!rc.isSuccess()) {
          logFatal("evqlbenchmark", "executing query failed: $0", rc.getMessage());
          ++errors;
        } else {
          ++requests_sent;
        }
        m.unlock();

        print(errors, requests_sent, global_start, stdout_os.get());
        if (errors > kMaxErrors || (has_max && requests_sent >= max_requests)) {
          break;
        }
      }

      evql_client_close(client);
    }));
  }

  for (auto& t : threads) {
    t.join();
  }

  print(errors, requests_sent, global_start, stdout_os.get());
  return 0;
}
<commit_msg>benchmark: cleanup<commit_after>/**
 * Copyright (c) 2016 DeepCortex GmbH <legal@eventql.io>
 * Authors:
 *   - Laura Schlimmer <laura@eventql.io>
 *
 * This program is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License ("the license") as
 * published by the Free Software Foundation, either version 3 of the License,
 * or any later version.
 *
 * In accordance with Section 7(e) of the license, the licensing of the Program
 * under the license does not imply a trademark license. Therefore any rights,
 * title and interest in our trademarks remain entirely with us.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the license for more details.
 *
 * You can be released from the requirements of the license by purchasing a
 * commercial license. Buying such a license is mandatory as soon as you develop
 * commercial activities involving this program without disclosing the source
 * code of your own applications
 */
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include <poll.h>
#include <thread>
#include "eventql/eventql.h"
#include "eventql/util/application.h"
#include "eventql/util/logging.h"
#include "eventql/util/UnixTime.h"
#include "eventql/util/return_code.h"
#include "eventql/util/cli/CLI.h"
#include "eventql/util/io/inputstream.h"
#include "eventql/util/cli/term.h"
#include "eventql/util/cli/flagparser.h"
#include "eventql/util/thread/threadpool.h"
#include "eventql/cli/cli_config.h"

struct TimeWindow {
  static constexpr auto kMillisPerWindow = 1 * kMillisPerSecond;

  TimeWindow() : num_requests_(0) {}

  void addRequest() {
    ++num_requests_;
  }

  int64_t getRemainingMillis() {
    UnixTime now;
    auto duration = now - start_;
    return kMillisPerWindow - duration.milliseconds();
  }

  void clear() {
    num_requests_ = 0;
    UnixTime now;
    start_ = now;
  }

protected:
  size_t num_requests_;
  UnixTime start_;
};

struct RequestStats {
  static constexpr auto kMaxErrors = 10;

  RequestStats() :
      failed_requests_(0),
      successful_requests_(0),
      max_requests_(0),
      has_max_(false) {}

  void addFailedRequest() {
    ++failed_requests_;
  }

  void addSuccessfulRequest() {
    ++successful_requests_;
  }

  uint64_t getFailedRequests() {
    return failed_requests_;
  }

  uint64_t getSuccessfulRequest() {
    return successful_requests_;
  }

  uint64_t getTotal() {
    return successful_requests_ + failed_requests_;
  }

  void setMaximumRequests(uint64_t max_requests) {
    max_requests_ = max_requests;
    has_max_ = true;
  }

  bool stop() {
    if (failed_requests_ >= kMaxErrors)
    if (!has_max_) {
      return false;
    }

    return getTotal() >= max_requests_;
  }

protected:
  uint64_t failed_requests_;
  uint64_t successful_requests_;
  uint64_t max_requests_;
  bool has_max_;
};

ReturnCode sendQuery(
    const String& query,
    const String& qry_db,
    evql_client_t* client) {
  uint64_t qry_flags = 0;
  if (!qry_db.empty()) {
    qry_flags |= EVQL_QUERY_SWITCHDB;
  }

  int rc = evql_query(client, query.c_str(), qry_db.c_str(), qry_flags);

  size_t result_ncols;
  if (rc == 0) {
    rc = evql_num_columns(client, &result_ncols);
  }

  while (rc >= 0) {
    const char** fields;
    size_t* field_lens;
    rc = evql_fetch_row(client, &fields, &field_lens);
    if (rc < 1) {
      break;
    }
  }

  evql_client_releasebuffers(client);

  if (rc == -1) {
    return ReturnCode::error("EIOERROR", evql_client_geterror(client));
  } else {
    return ReturnCode::success();
  }

}

void print(
    size_t num_errors,
    size_t num_succes,
    UnixTime start,
    OutputStream* stdout_os) {
  UnixTime now;
  auto duration = now - start;
  stdout_os->write(StringUtil::format(
    "total   successful    error      milliseconds\n"
    "   $0      $1            $2      $3\n\n",
    num_errors + num_succes,
    num_succes,
    num_errors,
    duration.milliseconds()
  ));
}

int main(int argc, const char** argv) {
  cli::FlagParser flags;

  flags.defineFlag(
      "help",
      cli::FlagParser::T_SWITCH,
      false,
      "?",
      NULL,
      "help",
      "<help>");

  flags.defineFlag(
      "version",
      cli::FlagParser::T_SWITCH,
      false,
      "v",
      NULL,
      "print version",
      "<switch>");

  flags.defineFlag(
      "host",
      cli::FlagParser::T_STRING,
      false,
      "h",
      NULL,
      "eventql server hostname",
      "<host>");

  flags.defineFlag(
      "port",
      cli::FlagParser::T_INTEGER,
      false,
      "p",
      NULL,
      "eventql server port",
      "<port>");

  flags.defineFlag(
      "database",
      cli::FlagParser::T_STRING,
      false,
      "d",
      NULL,
      "database",
      "<db>");

  flags.defineFlag(
      "query",
      cli::FlagParser::T_STRING,
      true,
      "q",
      NULL,
      "query str",
      "<query>");

  flags.defineFlag(
      "threads",
      cli::FlagParser::T_INTEGER,
      false,
      "t",
      "10",
      "number of threads",
      "<threads>");

  flags.defineFlag(
      "rate",
      cli::FlagParser::T_INTEGER,
      false,
      "r",
      "1",
      "number of requests per second",
      "<rate>");

  flags.defineFlag(
      "max",
      cli::FlagParser::T_INTEGER,
      false,
      "n",
      NULL,
      "number of requests per second",
      "<rate>");

  flags.defineFlag(
      "loglevel",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);
  Vector<String> cmd_argv = flags.getArgv();
  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  Application::init();
  Application::logToStderr("evqlbenchmark");
  auto stdin_is = InputStream::getStdin();
  auto stdout_os = OutputStream::getStdout();
  auto stderr_os = OutputStream::getStderr();

  bool print_help = flags.isSet("help");
  bool print_version = flags.isSet("version");
  if (print_version || print_help) {
    auto stdout_os = OutputStream::getStdout();
    stdout_os->write(
        StringUtil::format(
            "EventQL $0 ($1)\n"
            "Copyright (c) 2016, DeepCortex GmbH. All rights reserved.\n\n",
            eventql::kVersionString,
            eventql::kBuildID));
  }

  if (print_version) {
    return 1;
  }

  if (print_help) {
    stdout_os->write(
      "Usage: $ evqlbenchmark [OPTIONS] <command> [<args>]\n\n"
      "   -D, --database <db>       Select a database\n"
      "   -h, --host <hostname>     Set the EventQL server hostname\n"
      "   -p, --port <port>         Set the EventQL server port\n"
      "   -?, --help <topic>        Display a command's help text and exit\n"
      "   -v, --version             Display the version of this binary and exit\n\n"
      "evqlbenchmark commands:\n"
      );
    return 1;
  }

  logInfo("evqlbenchmark", "starting benchmark");
  /* options */
  eventql::ProcessConfigBuilder cfg_builder;
  {
    auto status = cfg_builder.loadDefaultConfigFile("evql");
    if (!status.isSuccess()) {
      logFatal("evqlbenchmark", "error while loading config file %s", status.message());
      return 0;
    }
  }

  if (flags.isSet("host")) {
    cfg_builder.setProperty("evql", "host", flags.getString("host"));
  }

  if (flags.isSet("port")) {
    cfg_builder.setProperty(
        "evql",
        "port",
        StringUtil::toString(flags.getInt("port")));
  }

  if (flags.isSet("database")) {
    cfg_builder.setProperty("evql", "database", flags.getString("database"));
  }

  eventql::cli::CLIConfig cfg(cfg_builder.getConfig());

  String qry_db;
  if (flags.isSet("database")) {
    qry_db = flags.getString("database");
  }

  auto qry_str = flags.getString("query");
  auto rate = flags.getInt("rate");

  const UnixTime global_start;

  std::mutex m;
  TimeWindow twindow;

  RequestStats rstats;
  if (flags.isSet("max")) {
    rstats.setMaximumRequests(flags.getInt("max"));
  }

  auto num_threads = flags.getInt("threads");
  Vector<std::thread> threads;
  for (size_t i = 0; i < num_threads; ++i) {
    threads.emplace_back(std::thread([&] () {

      /* connect to eventql client */
      auto client = evql_client_init();
      if (!client) {
        logFatal("evqlbenchmark", "can't initialize eventql client");
        return 0;
      }

      {
        auto rc = evql_client_connect(
            client,
            cfg.getHost().c_str(),
            cfg.getPort(),
            0);

        if (rc < 0) {
          logFatal("evqlbenchmark", "can't connect to eventql client: $0", evql_client_geterror(client));
          return 0;
        }
      }

      for (;;) {
        m.lock();
        /* check remaining time in current timewindow */
        if (twindow.getRemainingMillis() <= 0) {
          /* start a new timewindow */
          twindow.clear();
          m.unlock();
          continue;
        }
        m.unlock();

        /* send query */
        auto rc = sendQuery(qry_str, qry_db, client);
        m.lock();
        if (!rc.isSuccess()) {
          logFatal("evqlbenchmark", "executing query failed: $0", rc.getMessage());
          rstats.addFailedRequest();
        } else {
          rstats.addSuccessfulRequest();
        }
        m.unlock();

        //print(errors, requests_sent, global_start, stdout_os.get());
        if (rstats.stop()) {
          break;
        }
      }

      evql_client_close(client);
    }));
  }

  for (auto& t : threads) {
    t.join();
  }

  //print(errors, requests_sent, global_start, stdout_os.get());
  return 0;
}
<|endoftext|>


<commit_before>/*
* OCB Mode
* (C) 2013 Jack Lloyd
*
* Distributed under the terms of the Botan license
*/

#include <botan/ocb.h>
#include <botan/cmac.h>
#include <botan/internal/xor_buf.h>
#include <botan/internal/bit_ops.h>
#include <algorithm>

#include <botan/hex.h>
#include <iostream>

namespace Botan {

// Has to be in Botan namespace so unique_ptr can reference it
class L_computer
   {
   public:
      L_computer(const BlockCipher& cipher)
         {
         m_L_star.resize(cipher.block_size());
         cipher.encrypt(m_L_star);
         m_L_dollar = poly_double(star());
         m_L.push_back(poly_double(dollar()));
         }

      const secure_vector<byte>& star() const { return m_L_star; }

      const secure_vector<byte>& dollar() const { return m_L_dollar; }

      const secure_vector<byte>& operator()(size_t i) const { return get(i); }

      const secure_vector<byte>& get(size_t i) const
         {
         while(m_L.size() <= i)
            m_L.push_back(poly_double(m_L.back()));

         return m_L.at(i);
         }

   private:
      secure_vector<byte> poly_double(const secure_vector<byte>& in) const
         {
         return CMAC::poly_double(in, 0x87);
         }

      secure_vector<byte> m_L_dollar, m_L_star;
      mutable std::vector<secure_vector<byte>> m_L;
   };

#if 0
class Nonce_State
   {
   public:
      secure_vector<byte> update_nonce(const byte nonce[], size_t nonce_len);

      bool fresh_nonce() { bool b = false; std::swap(b, m_fresh); return b; }
   private:
      secure_vector<byte> m_stretch;
      bool m_fresh = false;
   };
#endif

namespace {

/*
* OCB's HASH
*/
secure_vector<byte> ocb_hash(const L_computer& L,
                             const BlockCipher& cipher,
                             const byte ad[], size_t ad_len)
   {
   const size_t BS = cipher.block_size();

   secure_vector<byte> sum(BS);
   secure_vector<byte> offset(BS);

   secure_vector<byte> buf(BS);

   const size_t ad_blocks = (ad_len / BS);
   const size_t ad_remainder = (ad_len % BS);

   for(size_t i = 0; i != ad_blocks; ++i)
      {
      // this loop could run in parallel
      offset ^= L(ctz(i+1));

      buf = offset;
      xor_buf(&buf[0], &ad[BS*i], BS);

      cipher.encrypt(buf);

      sum ^= buf;
      }

   if(ad_remainder)
      {
      offset ^= L.star();

      buf = offset;
      xor_buf(&buf[0], &ad[BS*ad_blocks], ad_remainder);
      buf[ad_len % BS] ^= 0x80;

      cipher.encrypt(buf);

      sum ^= buf;
      }

   return sum;
   }

}

OCB_Mode::OCB_Mode(BlockCipher* cipher, size_t tag_size, bool decrypting) :
   Buffered_Filter(cipher->parallel_bytes(), decrypting ? tag_size : 0),
   m_cipher(cipher), m_tag_size(tag_size),
   m_ad_hash(BS), m_offset(BS), m_checksum(BS)
   {
   if(m_cipher->block_size() != BS)
      throw std::invalid_argument("OCB requires a 128 bit cipher so cannot be used with " +
                                  m_cipher->name());

   if(m_tag_size != 16) // 64, 96 bits also supported
      throw std::invalid_argument("OCB cannot produce a " + std::to_string(m_tag_size) +
                                  " byte tag");

   }

OCB_Mode::~OCB_Mode() { /* for unique_ptr destructor */ }

bool OCB_Mode::valid_keylength(size_t n) const
   {
   return m_cipher->valid_keylength(n);
   }

std::string OCB_Mode::name() const
   {
   return m_cipher->name() + "/OCB"; // include tag size
   }

void OCB_Mode::set_key(const SymmetricKey& key)
   {
   m_cipher->set_key(key);
   m_L.reset(new L_computer(*m_cipher));
   }

void OCB_Mode::set_nonce(const byte nonce[], size_t nonce_len)
   {
   if(!valid_iv_length(nonce_len))
      throw Invalid_IV_Length(name(), nonce_len);

   byte bottom;
   secure_vector<byte> stretch;

   if(1) // need to recompute stretch (save iv to compare)
      {
      secure_vector<byte> buf(BS);

      const size_t offset = BS - nonce_len;

      copy_mem(&buf[offset], nonce, nonce_len);
      buf[offset-1] = 1;

      bottom = buf[15] & 0x3F;
      buf[15] &= 0xC0;

      m_cipher->encrypt(buf);

      for(size_t i = 0; i != 8; ++i)
         buf.push_back(buf[i] ^ buf[i+1]);

      stretch = buf;
      }

   // now set the offset from stretch and bottom

   const size_t shift_bytes = bottom / 8;
   const size_t shift_bits  = bottom % 8;

   for(size_t i = 0; i != BS; ++i)
      {
      m_offset[i]  = (stretch[i+shift_bytes] << shift_bits);
      m_offset[i] |= (stretch[i+shift_bytes+1] >> (8-shift_bits));
      }
   }

void OCB_Mode::start_msg()
   {
   //BOTAN_ASSERT(m_nonce_state.fresh_nonce(), "Nonce state is fresh");
   }

void OCB_Mode::set_associated_data(const byte ad[], size_t ad_len)
   {
   m_ad_hash = ocb_hash(*m_L, *m_cipher, &ad[0], ad_len);
   }

void OCB_Mode::write(const byte input[], size_t length)
   {
   Buffered_Filter::write(input, length);
   }

void OCB_Mode::end_msg()
   {
   Buffered_Filter::end_msg();
   }

void OCB_Encryption::buffered_block(const byte input[], size_t input_length)
   {
   BOTAN_ASSERT(input_length % BS == 0, "Input length is an even number of blocks");

   const size_t blocks = input_length / BS;

   const size_t par_bytes = m_cipher->parallel_bytes();

   BOTAN_ASSERT(par_bytes % BS == 0, "Cipher is parallel in full blocks");

   const size_t par_blocks = par_bytes / BS;

   const L_computer& L = *m_L; // convenient name

   secure_vector<byte> ctext_buf(par_bytes);
   secure_vector<byte> csum_accum(par_bytes);
   secure_vector<byte> offsets(par_bytes);

   size_t blocks_left = blocks;

   while(blocks_left)
      {
      const size_t to_proc = std::min(blocks_left, par_blocks);
      const size_t proc_bytes = to_proc * BS;

      xor_buf(&csum_accum[0], &input[0], proc_bytes);

      for(size_t i = 0; i != to_proc; ++i)
         {
         m_offset ^= L(ctz(++m_block_index));
         copy_mem(&offsets[BS*i], &m_offset[0], BS);
         }

      copy_mem(&ctext_buf[0], &input[0], proc_bytes);

      ctext_buf ^= offsets;
      m_cipher->encrypt(ctext_buf);
      ctext_buf ^= offsets;

      send(ctext_buf, proc_bytes);

      input += proc_bytes;
      blocks_left -= to_proc;
      }

   // fold into checksum
   for(size_t i = 0; i != csum_accum.size(); ++i)
      m_checksum[i % BS] ^= csum_accum[i];
   }

void OCB_Encryption::buffered_final(const byte input[], size_t input_length)
   {
   if(input_length >= BS)
      {
      const size_t final_blocks = input_length / BS;
      const size_t final_bytes = final_blocks * BS;
      buffered_block(input, final_bytes);
      input += final_bytes;
      input_length -= final_bytes;
      }

   if(input_length)
      {
      BOTAN_ASSERT(input_length < BS, "Only a partial block left");

      xor_buf(&m_checksum[0], &input[0], input_length);
      m_checksum[input_length] ^= 0x80;

      m_offset ^= m_L->star(); // Offset_*

      secure_vector<byte> buf(BS);
      m_cipher->encrypt(m_offset, buf);
      xor_buf(&buf[0], &input[0], input_length);

      send(buf, input_length); // final ciphertext
      }

   // now compute the tag
   secure_vector<byte> mac = m_offset;
   mac ^= m_checksum;
   mac ^= m_L->dollar();

   m_cipher->encrypt(mac);

   mac ^= m_ad_hash;

   send(mac);

   zeroise(m_checksum);
   zeroise(m_offset);
   m_block_index = 0;
   }

void OCB_Decryption::buffered_block(const byte input[], size_t input_length)
   {
   BOTAN_ASSERT(input_length % BS == 0, "Input length is an even number of blocks");

   const size_t blocks = input_length / BS;

   const L_computer& L = *m_L;

   secure_vector<byte> ptext_buf(BS);

   for(size_t i = 0; i != blocks; ++i)
      {
      // could run in parallel

      m_offset ^= L(ctz(++m_block_index));

      ptext_buf = m_offset;
      xor_buf(&ptext_buf[0], &input[BS*i], BS);
      m_cipher->decrypt(ptext_buf);
      ptext_buf ^= m_offset;

      send(ptext_buf);
      m_checksum ^= ptext_buf;
      }
   }

void OCB_Decryption::buffered_final(const byte input[], size_t input_length)
   {
   BOTAN_ASSERT(input_length >= m_tag_size, "We have the tag");

   const byte* included_tag = &input[input_length-m_tag_size];
   input_length -= m_tag_size;

   if(input_length >= BS)
      {
      const size_t final_blocks = input_length / BS;
      const size_t final_bytes = final_blocks * BS;
      buffered_block(input, final_bytes);
      input += final_bytes;
      input_length -= final_bytes;
      }

   if(input_length)
      {
      BOTAN_ASSERT(input_length < BS, "Only a partial block left");

      m_offset ^= m_L->star(); // Offset_*

      secure_vector<byte> buf(BS);
      m_cipher->encrypt(m_offset, buf); // P_*

      xor_buf(&buf[0], &input[0], input_length);

      xor_buf(&m_checksum[0], &buf[0], input_length);
      m_checksum[input_length] ^= 0x80;

      send(buf, input_length); // final plaintext
      }

   // now compute the tag
   secure_vector<byte> mac = m_offset;
   mac ^= m_checksum;
   mac ^= m_L->dollar();

   m_cipher->encrypt(mac);

   mac ^= m_ad_hash;

   zeroise(m_checksum);
   zeroise(m_offset);
   m_block_index = 0;

   if(!same_mem(&mac[0], included_tag, m_tag_size))
      throw Integrity_Failure("OCB tag check failed");
   }

}
<commit_msg>Parallel OCB decryption<commit_after>/*
* OCB Mode
* (C) 2013 Jack Lloyd
*
* Distributed under the terms of the Botan license
*/

#include <botan/ocb.h>
#include <botan/cmac.h>
#include <botan/internal/xor_buf.h>
#include <botan/internal/bit_ops.h>
#include <algorithm>

#include <botan/hex.h>
#include <iostream>

namespace Botan {

// Has to be in Botan namespace so unique_ptr can reference it
class L_computer
   {
   public:
      L_computer(const BlockCipher& cipher)
         {
         m_L_star.resize(cipher.block_size());
         cipher.encrypt(m_L_star);
         m_L_dollar = poly_double(star());
         m_L.push_back(poly_double(dollar()));
         }

      const secure_vector<byte>& star() const { return m_L_star; }

      const secure_vector<byte>& dollar() const { return m_L_dollar; }

      const secure_vector<byte>& operator()(size_t i) const { return get(i); }

      const secure_vector<byte>& get(size_t i) const
         {
         while(m_L.size() <= i)
            m_L.push_back(poly_double(m_L.back()));

         return m_L.at(i);
         }

   private:
      secure_vector<byte> poly_double(const secure_vector<byte>& in) const
         {
         return CMAC::poly_double(in, 0x87);
         }

      secure_vector<byte> m_L_dollar, m_L_star;
      mutable std::vector<secure_vector<byte>> m_L;
   };

#if 0
class Nonce_State
   {
   public:
      secure_vector<byte> update_nonce(const byte nonce[], size_t nonce_len);

      bool fresh_nonce() { bool b = false; std::swap(b, m_fresh); return b; }
   private:
      secure_vector<byte> m_stretch;
      bool m_fresh = false;
   };
#endif

namespace {

/*
* OCB's HASH
*/
secure_vector<byte> ocb_hash(const L_computer& L,
                             const BlockCipher& cipher,
                             const byte ad[], size_t ad_len)
   {
   const size_t BS = cipher.block_size();

   secure_vector<byte> sum(BS);
   secure_vector<byte> offset(BS);

   secure_vector<byte> buf(BS);

   const size_t ad_blocks = (ad_len / BS);
   const size_t ad_remainder = (ad_len % BS);

   for(size_t i = 0; i != ad_blocks; ++i)
      {
      // this loop could run in parallel
      offset ^= L(ctz(i+1));

      buf = offset;
      xor_buf(&buf[0], &ad[BS*i], BS);

      cipher.encrypt(buf);

      sum ^= buf;
      }

   if(ad_remainder)
      {
      offset ^= L.star();

      buf = offset;
      xor_buf(&buf[0], &ad[BS*ad_blocks], ad_remainder);
      buf[ad_len % BS] ^= 0x80;

      cipher.encrypt(buf);

      sum ^= buf;
      }

   return sum;
   }

}

OCB_Mode::OCB_Mode(BlockCipher* cipher, size_t tag_size, bool decrypting) :
   Buffered_Filter(cipher->parallel_bytes(), decrypting ? tag_size : 0),
   m_cipher(cipher), m_tag_size(tag_size),
   m_ad_hash(BS), m_offset(BS), m_checksum(BS)
   {
   if(m_cipher->block_size() != BS)
      throw std::invalid_argument("OCB requires a 128 bit cipher so cannot be used with " +
                                  m_cipher->name());

   if(m_tag_size != 16) // 64, 96 bits also supported
      throw std::invalid_argument("OCB cannot produce a " + std::to_string(m_tag_size) +
                                  " byte tag");

   }

OCB_Mode::~OCB_Mode() { /* for unique_ptr destructor */ }

bool OCB_Mode::valid_keylength(size_t n) const
   {
   return m_cipher->valid_keylength(n);
   }

std::string OCB_Mode::name() const
   {
   return m_cipher->name() + "/OCB"; // include tag size
   }

void OCB_Mode::set_key(const SymmetricKey& key)
   {
   m_cipher->set_key(key);
   m_L.reset(new L_computer(*m_cipher));
   }

void OCB_Mode::set_nonce(const byte nonce[], size_t nonce_len)
   {
   if(!valid_iv_length(nonce_len))
      throw Invalid_IV_Length(name(), nonce_len);

   byte bottom;
   secure_vector<byte> stretch;

   if(1) // need to recompute stretch (save iv to compare)
      {
      secure_vector<byte> buf(BS);

      const size_t offset = BS - nonce_len;

      copy_mem(&buf[offset], nonce, nonce_len);
      buf[offset-1] = 1;

      bottom = buf[15] & 0x3F;
      buf[15] &= 0xC0;

      m_cipher->encrypt(buf);

      for(size_t i = 0; i != 8; ++i)
         buf.push_back(buf[i] ^ buf[i+1]);

      stretch = buf;
      }

   // now set the offset from stretch and bottom

   const size_t shift_bytes = bottom / 8;
   const size_t shift_bits  = bottom % 8;

   for(size_t i = 0; i != BS; ++i)
      {
      m_offset[i]  = (stretch[i+shift_bytes] << shift_bits);
      m_offset[i] |= (stretch[i+shift_bytes+1] >> (8-shift_bits));
      }
   }

void OCB_Mode::start_msg()
   {
   //BOTAN_ASSERT(m_nonce_state.fresh_nonce(), "Nonce state is fresh");
   }

void OCB_Mode::set_associated_data(const byte ad[], size_t ad_len)
   {
   m_ad_hash = ocb_hash(*m_L, *m_cipher, &ad[0], ad_len);
   }

void OCB_Mode::write(const byte input[], size_t length)
   {
   Buffered_Filter::write(input, length);
   }

void OCB_Mode::end_msg()
   {
   Buffered_Filter::end_msg();
   }

void OCB_Encryption::buffered_block(const byte input[], size_t input_length)
   {
   BOTAN_ASSERT(input_length % BS == 0, "Input length is an even number of blocks");

   const size_t blocks = input_length / BS;

   const size_t par_bytes = m_cipher->parallel_bytes();

   BOTAN_ASSERT(par_bytes % BS == 0, "Cipher is parallel in full blocks");

   const size_t par_blocks = par_bytes / BS;

   const L_computer& L = *m_L; // convenient name

   secure_vector<byte> ctext_buf(par_bytes);
   secure_vector<byte> csum_accum(par_bytes);
   secure_vector<byte> offsets(par_bytes);

   size_t blocks_left = blocks;

   while(blocks_left)
      {
      const size_t to_proc = std::min(blocks_left, par_blocks);
      const size_t proc_bytes = to_proc * BS;

      xor_buf(&csum_accum[0], &input[0], proc_bytes);

      for(size_t i = 0; i != to_proc; ++i)
         {
         m_offset ^= L(ctz(++m_block_index));
         copy_mem(&offsets[BS*i], &m_offset[0], BS);
         }

      copy_mem(&ctext_buf[0], &input[0], proc_bytes);

      ctext_buf ^= offsets;
      m_cipher->encrypt(ctext_buf);
      ctext_buf ^= offsets;

      send(ctext_buf, proc_bytes);

      input += proc_bytes;
      blocks_left -= to_proc;
      }

   // fold into checksum
   for(size_t i = 0; i != csum_accum.size(); ++i)
      m_checksum[i % BS] ^= csum_accum[i];
   }

void OCB_Encryption::buffered_final(const byte input[], size_t input_length)
   {
   if(input_length >= BS)
      {
      const size_t final_blocks = input_length / BS;
      const size_t final_bytes = final_blocks * BS;
      buffered_block(input, final_bytes);
      input += final_bytes;
      input_length -= final_bytes;
      }

   if(input_length)
      {
      BOTAN_ASSERT(input_length < BS, "Only a partial block left");

      xor_buf(&m_checksum[0], &input[0], input_length);
      m_checksum[input_length] ^= 0x80;

      m_offset ^= m_L->star(); // Offset_*

      secure_vector<byte> buf(BS);
      m_cipher->encrypt(m_offset, buf);
      xor_buf(&buf[0], &input[0], input_length);

      send(buf, input_length); // final ciphertext
      }

   // now compute the tag
   secure_vector<byte> mac = m_offset;
   mac ^= m_checksum;
   mac ^= m_L->dollar();

   m_cipher->encrypt(mac);

   mac ^= m_ad_hash;

   send(mac);

   zeroise(m_checksum);
   zeroise(m_offset);
   m_block_index = 0;
   }

void OCB_Decryption::buffered_block(const byte input[], size_t input_length)
   {
   BOTAN_ASSERT(input_length % BS == 0, "Input length is an even number of blocks");

   const size_t blocks = input_length / BS;

   const size_t par_bytes = m_cipher->parallel_bytes();

   BOTAN_ASSERT(par_bytes % BS == 0, "Cipher is parallel in full blocks");

   const size_t par_blocks = par_bytes / BS;

   const L_computer& L = *m_L; // convenient name

   secure_vector<byte> ptext_buf(par_bytes);
   secure_vector<byte> csum_accum(par_bytes);
   secure_vector<byte> offsets(par_bytes);

   size_t blocks_left = blocks;

   while(blocks_left)
      {
      const size_t to_proc = std::min(blocks_left, par_blocks);
      const size_t proc_bytes = to_proc * BS;

      for(size_t i = 0; i != to_proc; ++i)
         {
         m_offset ^= L(ctz(++m_block_index));
         copy_mem(&offsets[BS*i], &m_offset[0], BS);
         }

      copy_mem(&ptext_buf[0], &input[0], proc_bytes);

      ptext_buf ^= offsets;
      m_cipher->decrypt(ptext_buf);
      ptext_buf ^= offsets;

      xor_buf(&csum_accum[0], &ptext_buf[0], proc_bytes);

      send(ptext_buf, proc_bytes);

      input += proc_bytes;
      blocks_left -= to_proc;
      }

   // fold into checksum
   for(size_t i = 0; i != csum_accum.size(); ++i)
      m_checksum[i % BS] ^= csum_accum[i];
   }

void OCB_Decryption::buffered_final(const byte input[], size_t input_length)
   {
   BOTAN_ASSERT(input_length >= m_tag_size, "We have the tag");

   const byte* included_tag = &input[input_length-m_tag_size];
   input_length -= m_tag_size;

   if(input_length >= BS)
      {
      const size_t final_blocks = input_length / BS;
      const size_t final_bytes = final_blocks * BS;
      buffered_block(input, final_bytes);
      input += final_bytes;
      input_length -= final_bytes;
      }

   if(input_length)
      {
      BOTAN_ASSERT(input_length < BS, "Only a partial block left");

      m_offset ^= m_L->star(); // Offset_*

      secure_vector<byte> buf(BS);
      m_cipher->encrypt(m_offset, buf); // P_*

      xor_buf(&buf[0], &input[0], input_length);

      xor_buf(&m_checksum[0], &buf[0], input_length);
      m_checksum[input_length] ^= 0x80;

      send(buf, input_length); // final plaintext
      }

   // now compute the tag
   secure_vector<byte> mac = m_offset;
   mac ^= m_checksum;
   mac ^= m_L->dollar();

   m_cipher->encrypt(mac);

   mac ^= m_ad_hash;

   zeroise(m_checksum);
   zeroise(m_offset);
   m_block_index = 0;

   if(!same_mem(&mac[0], included_tag, m_tag_size))
      throw Integrity_Failure("OCB tag check failed");
   }

}
<|endoftext|>


<commit_before>/****************************************************************/
/*       PIKA - Phase field snow micro-structure model          */
/*                                                              */
/*          Prepared by Battelle Energy Alliance, LLC           */
/*            Under Contract No. DE-AC07-05ID14517              */
/*            With the U. S. Department of Energy               */
/*                                                              */
/*            See COPYRIGHT for full restrictions               */
/****************************************************************/

#include "PhaseTransition.h"
#include "AirProperties.h"

template<>
InputParameters validParams<PhaseTransition>()
{
  InputParameters params = validParams<ACBulk>();
  params += validParams<CoefficientKernelInterface>();
  params.addRequiredCoupledVar("chemical_potential", "The chemical potential variable to couple");
  params.addParam<std::string>("lambda", "lambda", "The name of the material property containing the definition of lambda");
  params.addParam<std::string>("equilibrium_concentration", "equilibrium_concentration", "The name of the material property containing the equilibrium concentration");

  return params;
}

PhaseTransition::PhaseTransition(const std::string & name, InputParameters parameters) :
    ACBulk(name, parameters),
    PropertyUserObjectInterface(name, parameters),
    CoefficientKernelInterface(name, parameters),
    _s(coupledValue("chemical_potential")),
    _lambda(getMaterialProperty<Real>(getParam<std::string>("lambda"))),
    _s_eq(getMaterialProperty<Real>(getParam<std::string>("equilibrium_concentration")))
{
}

Real
PhaseTransition::computeDFDOP(PFFunctionType type)
{
  switch (type)
  {
    case Residual:
     return - coefficient(_qp) * (_lambda[_qp]) * (_s[_qp] - _s_eq[_qp]) * (1.0 - _u[_qp]*_u[_qp])*(1.0 - _u[_qp]*_u[_qp]);

    case Jacobian:
      return coefficient(_qp) *  4.0 * _lambda[_qp] * _u[_qp] * (-_u[_qp] * _u[_qp]+1.0) * (_s[_qp] - _s_eq[_qp]) * _phi[_j][_qp];
  }
}
<commit_msg>Always have return value<commit_after>/****************************************************************/
/*       PIKA - Phase field snow micro-structure model          */
/*                                                              */
/*          Prepared by Battelle Energy Alliance, LLC           */
/*            Under Contract No. DE-AC07-05ID14517              */
/*            With the U. S. Department of Energy               */
/*                                                              */
/*            See COPYRIGHT for full restrictions               */
/****************************************************************/

#include "PhaseTransition.h"
#include "AirProperties.h"

template<>
InputParameters validParams<PhaseTransition>()
{
  InputParameters params = validParams<ACBulk>();
  params += validParams<CoefficientKernelInterface>();
  params.addRequiredCoupledVar("chemical_potential", "The chemical potential variable to couple");
  params.addParam<std::string>("lambda", "lambda", "The name of the material property containing the definition of lambda");
  params.addParam<std::string>("equilibrium_concentration", "equilibrium_concentration", "The name of the material property containing the equilibrium concentration");

  return params;
}

PhaseTransition::PhaseTransition(const std::string & name, InputParameters parameters) :
    ACBulk(name, parameters),
    PropertyUserObjectInterface(name, parameters),
    CoefficientKernelInterface(name, parameters),
    _s(coupledValue("chemical_potential")),
    _lambda(getMaterialProperty<Real>(getParam<std::string>("lambda"))),
    _s_eq(getMaterialProperty<Real>(getParam<std::string>("equilibrium_concentration")))
{
}

Real
PhaseTransition::computeDFDOP(PFFunctionType type)
{
  switch (type)
  {
    case Residual:
     return - coefficient(_qp) * (_lambda[_qp]) * (_s[_qp] - _s_eq[_qp]) * (1.0 - _u[_qp]*_u[_qp])*(1.0 - _u[_qp]*_u[_qp]);

    case Jacobian:
      return coefficient(_qp) *  4.0 * _lambda[_qp] * _u[_qp] * (-_u[_qp] * _u[_qp]+1.0) * (_s[_qp] - _s_eq[_qp]) * _phi[_j][_qp];
  }
  return 0.0;
}
<|endoftext|>


<commit_before>#include "gboost.h"
#include "solver.h"
#include "core/tpool.h"
#include "core/logger.h"
#include "gboost_stump.h"
#include "core/ibstream.h"
#include "core/obstream.h"

using namespace nano;

static auto measure(const task_t& task, const fold_t& fold, const tensor4d_t& outputs, const loss_t& loss)
{
        const auto& tpool = tpool_t::instance();

        std::vector<stats_t> errors(tpool.workers());
        std::vector<stats_t> values(tpool.workers());

        loopit(task.size(fold), [&] (const size_t i, const size_t t)
        {
                assert(t < tpool.workers());
                const auto input = task.input(fold, i);
                const auto target = task.target(fold, i);
                const auto output = outputs.tensor(i);

                errors[t](loss.error(target, output));
                values[t](loss.value(target, output));
        });

        for (size_t t = 1; t < tpool.workers(); ++ t)
        {
                errors[0](errors[t]);
                values[0](values[t]);
        }

        return std::make_pair(errors[0], values[0]);
}

static void update(const task_t& task, const fold_t& fold, tensor4d_t& outputs, const stump_t& stump)
{
        loopi(task.size(fold), [&] (const size_t i)
        {
                const auto input = task.input(fold, i);
                const auto oindex = input(stump.m_feature) < stump.m_threshold ? 0 : 1;
                outputs.array(i) += stump.m_outputs.array(oindex);
        });
}

// todo: break the computation in smaller functions (move them to gboost.h)
// todo: better comment tensor/tensor.h

void gboost_stump_t::to_json(json_t& json) const
{
        nano::to_json(json,
                "rounds", m_rounds,
                "patience", m_patience,
                "stump", m_stype, "stumps", join(enum_values<stump>()),
                "solver", m_solver, "solvers", join(get_solvers().ids()),
                "regularization", m_rtype, "regularizations", join(enum_values<regularization>()));
}

void gboost_stump_t::from_json(const json_t& json)
{
        nano::from_json(json,
                "rounds", m_rounds,
                "patience", m_patience,
                "stump", m_stype,
                "solver", m_solver,
                "regularization", m_rtype);
}

trainer_result_t gboost_stump_t::train(const task_t& task, const size_t fold, const loss_t& loss)
{
        // check if the solver is properly set
        rsolver_t solver;
        critical(solver = get_solvers().get(m_solver),
                strcat("invalid solver (", m_solver, ")"));

        m_idims = task.idims();
        m_odims = task.odims();

        m_stumps.clear();

        trainer_result_t result("<config>");
        timer_t timer;

        const auto fold_train = fold_t{fold, protocol::train};
        const auto fold_valid = fold_t{fold, protocol::valid};
        const auto fold_test = fold_t{fold, protocol::test};

        tensor4d_t outputs_train(cat_dims(task.size(fold_train), m_odims));
        tensor4d_t outputs_valid(cat_dims(task.size(fold_valid), m_odims));
        tensor4d_t outputs_test(cat_dims(task.size(fold_test), m_odims));

        outputs_train.zero();
        outputs_valid.zero();
        outputs_test.zero();

        stats_t errors_train, errors_valid, errors_test;
        stats_t values_train, values_valid, values_test;

        std::tie(errors_train, values_train) = ::measure(task, fold_train, outputs_train, loss);
        std::tie(errors_valid, values_valid) = ::measure(task, fold_valid, outputs_valid, loss);
        std::tie(errors_test, values_test) = ::measure(task, fold_test, outputs_test, loss);

        result.update(trainer_state_t{timer.milliseconds(), 0,
                {values_train.avg(), errors_train.avg()},
                {values_valid.avg(), errors_valid.avg()},
                {values_test.avg(), errors_test.avg()}},
                m_patience);

        log_info() << result;

        tensor4d_t residuals_train(cat_dims(task.size(fold_train), m_odims));
        tensor3d_t residuals_pos1(m_odims), residuals_pos2(m_odims);
        tensor3d_t residuals_neg1(m_odims), residuals_neg2(m_odims);

        stump_t stump;
        tensor4d_t stump_outputs_train(cat_dims(task.size(fold_train), m_odims));

        tensor4d_t targets(cat_dims(task.size(fold_train), m_odims));
        for (size_t i = 0, size = task.size(fold_train); i < size; ++ i)
        {
                const auto target = task.target(fold_train, i);
                targets.tensor(i) = target.tensor();
        }

        gboost_lsearch_function_t func(targets, outputs_train, stump_outputs_train, loss);

        for (auto round = 0; round < m_rounds && !result.is_done(); ++ round)
        {
                for (size_t i = 0, size = task.size(fold_train); i < size; ++ i)
                {
                        const auto input = task.input(fold_train, i);
                        const auto target = task.target(fold_train, i);
                        const auto output = outputs_train.tensor(i);

                        const auto vgrad = loss.vgrad(target, output);
                        residuals_train.vector(i) = -vgrad.vector();
                }

                scalar_t best_value = std::numeric_limits<scalar_t>::max();

                // todo: generalize this - e.g. to use features that are products of two input features
                for (auto feature = 0; feature < nano::size(m_idims); ++ feature)
                {
                        scalars_t fvalues(task.size(fold_train));
                        for (size_t i = 0, size = task.size(fold_train); i < size; ++ i)
                        {
                                const auto input = task.input(fold_train, i);
                                fvalues[i] = input(feature);
                        }

                        auto thresholds = fvalues;
                        std::sort(thresholds.begin(), thresholds.end());
                        thresholds.erase(std::unique(thresholds.begin(), thresholds.end()), thresholds.end());

                        for (size_t t = 0; t + 1 < thresholds.size(); ++ t)
                        {
                                const auto threshold = (thresholds[t + 0] + thresholds[t + 1]) / 2;

                                residuals_pos1.zero(), residuals_pos2.zero();
                                residuals_neg1.zero(), residuals_neg2.zero();

                                int cnt_pos = 0, cnt_neg = 0;
                                for (size_t i = 0, size = task.size(fold_train); i < size; ++ i)
                                {
                                        const auto residual = residuals_train.tensor(i).array();
                                        if (fvalues[i] < threshold)
                                        {
                                                cnt_neg ++;
                                                residuals_neg1.array() += residual;
                                                residuals_neg2.array() += residual * residual;
                                        }
                                        else
                                        {
                                                cnt_pos ++;
                                                residuals_pos1.array() += residual;
                                                residuals_pos2.array() += residual * residual;
                                        }
                                }

                                const auto value =
                                        (residuals_pos2.array().sum() - residuals_pos1.array().square().sum() / cnt_pos) +
                                        (residuals_neg2.array().sum() - residuals_neg1.array().square().sum() / cnt_neg);

                                //log_info() << "feature = " << feature
                                //        << ", threshold = " << threshold
                                //        << ", value = " << value
                                //        << ", count = " << cnt_neg << "+" << cnt_pos << "=" << task.size(fold_train);

                                if (value < best_value)
                                {
                                        best_value = value;
                                        stump.m_feature = feature;
                                        stump.m_threshold = threshold;
                                        stump.m_outputs.resize(cat_dims(2, m_odims));
                                        stump.m_outputs.vector(0) = residuals_neg1.vector() / cnt_neg;
                                        stump.m_outputs.vector(1) = residuals_pos1.vector() / cnt_pos;
                                }

                                // todo: fit both real and discrete stumps
                        }
                }

                // line-search
                for (size_t i = 0, size = task.size(fold_train); i < size; ++ i)
                {
                        const auto input = task.input(fold_train, i);
                        const auto oindex = input(stump.m_feature) < stump.m_threshold ? 0 : 1;
                        stump_outputs_train.tensor(i) = stump.m_outputs.tensor(oindex);
                }

                const auto state = solver->minimize(100, epsilon2<scalar_t>(), func, vector_t::Constant(1, 0));
                const auto step = state.x(0);

                stump.m_outputs.vector() *= step;
                m_stumps.push_back(stump);

                // update current outputs
                update(task, fold_train, outputs_train, stump);
                update(task, fold_valid, outputs_valid, stump);
                update(task, fold_test, outputs_test, stump);

                std::tie(errors_train, values_train) = ::measure(task, fold_train, outputs_train, loss);
                std::tie(errors_valid, values_valid) = ::measure(task, fold_valid, outputs_valid, loss);
                std::tie(errors_test, values_test) = ::measure(task, fold_test, outputs_test, loss);

                result.update(trainer_state_t{timer.milliseconds(), round + 1,
                        {values_train.avg(), errors_train.avg()},
                        {values_valid.avg(), errors_valid.avg()},
                        {values_test.avg(), errors_test.avg()}},
                        m_patience);

                log_info() << result << ",feature=" << stump.m_feature << ",gamma=" << step
                        << ",solver=" << state.m_status
                        << "|i=" << state.m_iterations << "|f=" << state.f << "|g=" << state.convergence_criteria() << ".";
        }

        // keep only the stumps up to optimum epoch (on the validation dataset)
        m_stumps.erase(
                m_stumps.begin() + result.optimum().m_epoch,
                m_stumps.end());

        return result;
}

tensor3d_t gboost_stump_t::output(const tensor3d_t& input) const
{
        assert(input.dims() == m_idims);

        tensor3d_t output(m_odims);
        output.zero();

        const auto idata = input.array();
        auto odata = output.array();

        for (const auto& stump : m_stumps)
        {
                const auto oindex = idata(stump.m_feature) < stump.m_threshold ? 0 : 1;
                odata.array() += stump.m_outputs.array(oindex);
        }

        return output;
}

bool gboost_stump_t::save(obstream_t& stream) const
{
        if (    !stream.write(m_idims) ||
                !stream.write(m_odims) ||
                !stream.write(m_rounds) ||
                !stream.write(m_stype) ||
                !stream.write(m_rtype) ||
                !stream.write(m_stumps.size()))
        {
                return false;
        }

        for (const auto& stump : m_stumps)
        {
                assert(stump.m_feature >= 0 && stump.m_feature < nano::size(m_idims));
                assert(stump.m_outputs.dims() == cat_dims(2, m_odims));

                if (    !stream.write(stump.m_feature) ||
                        !stream.write(stump.m_threshold) ||
                        !stream.write_tensor(stump.m_outputs))
                {
                       return false;
                }
        }

        return true;
}

bool gboost_stump_t::load(ibstream_t& stream)
{
        size_t n_stumps = 0;
        if (    !stream.read(m_idims) ||
                !stream.read(m_odims) ||
                !stream.read(m_rounds) ||
                !stream.read(m_stype) ||
                !stream.read(m_rtype) ||
                !stream.read(n_stumps))
        {
                return false;
        }

        m_stumps.resize(n_stumps);
        for (auto& stump : m_stumps)
        {
                if (    !stream.read(stump.m_feature) ||
                        !stream.read(stump.m_threshold) ||
                        !stream.read_tensor(stump.m_outputs) ||
                        stump.m_feature < 0 ||
                        stump.m_feature >= nano::size(m_idims) ||
                        stump.m_outputs.dims() != cat_dims(2, m_odims))
                {
                        return false;
                }
        }

        // todo: more verbose loading (#stumps, feature or coefficient statistics, idims...)

        return true;
}

probes_t gboost_stump_t::probes() const
{
        // todo: add probes here to measure the training and the evaluation time
        probes_t probes;
        return probes;
}
<commit_msg>improve logging for gboost stump<commit_after>#include "gboost.h"
#include "solver.h"
#include "core/tpool.h"
#include "core/logger.h"
#include "gboost_stump.h"
#include "core/ibstream.h"
#include "core/obstream.h"

using namespace nano;

static auto measure(const task_t& task, const fold_t& fold, const tensor4d_t& outputs, const loss_t& loss)
{
        const auto& tpool = tpool_t::instance();

        std::vector<stats_t> errors(tpool.workers());
        std::vector<stats_t> values(tpool.workers());

        loopit(task.size(fold), [&] (const size_t i, const size_t t)
        {
                assert(t < tpool.workers());
                const auto input = task.input(fold, i);
                const auto target = task.target(fold, i);
                const auto output = outputs.tensor(i);

                errors[t](loss.error(target, output));
                values[t](loss.value(target, output));
        });

        for (size_t t = 1; t < tpool.workers(); ++ t)
        {
                errors[0](errors[t]);
                values[0](values[t]);
        }

        return std::make_pair(errors[0], values[0]);
}

static void update(const task_t& task, const fold_t& fold, tensor4d_t& outputs, const stump_t& stump)
{
        loopi(task.size(fold), [&] (const size_t i)
        {
                const auto input = task.input(fold, i);
                const auto oindex = input(stump.m_feature) < stump.m_threshold ? 0 : 1;
                outputs.array(i) += stump.m_outputs.array(oindex);
        });
}

// todo: break the computation in smaller functions (move them to gboost.h)
// todo: better comment tensor/tensor.h

void gboost_stump_t::to_json(json_t& json) const
{
        nano::to_json(json,
                "rounds", m_rounds,
                "patience", m_patience,
                "stump", m_stype, "stumps", join(enum_values<stump>()),
                "solver", m_solver, "solvers", join(get_solvers().ids()),
                "regularization", m_rtype, "regularizations", join(enum_values<regularization>()));
}

void gboost_stump_t::from_json(const json_t& json)
{
        nano::from_json(json,
                "rounds", m_rounds,
                "patience", m_patience,
                "stump", m_stype,
                "solver", m_solver,
                "regularization", m_rtype);
}

trainer_result_t gboost_stump_t::train(const task_t& task, const size_t fold, const loss_t& loss)
{
        // check if the solver is properly set
        rsolver_t solver;
        critical(solver = get_solvers().get(m_solver),
                strcat("search solver (", m_solver, ")."));

        m_idims = task.idims();
        m_odims = task.odims();

        m_stumps.clear();

        trainer_result_t result("<config>");
        timer_t timer;

        const auto fold_train = fold_t{fold, protocol::train};
        const auto fold_valid = fold_t{fold, protocol::valid};
        const auto fold_test = fold_t{fold, protocol::test};

        tensor4d_t outputs_train(cat_dims(task.size(fold_train), m_odims));
        tensor4d_t outputs_valid(cat_dims(task.size(fold_valid), m_odims));
        tensor4d_t outputs_test(cat_dims(task.size(fold_test), m_odims));

        outputs_train.zero();
        outputs_valid.zero();
        outputs_test.zero();

        stats_t errors_train, errors_valid, errors_test;
        stats_t values_train, values_valid, values_test;

        std::tie(errors_train, values_train) = ::measure(task, fold_train, outputs_train, loss);
        std::tie(errors_valid, values_valid) = ::measure(task, fold_valid, outputs_valid, loss);
        std::tie(errors_test, values_test) = ::measure(task, fold_test, outputs_test, loss);

        result.update(trainer_state_t{timer.milliseconds(), 0,
                {values_train.avg(), errors_train.avg()},
                {values_valid.avg(), errors_valid.avg()},
                {values_test.avg(), errors_test.avg()}},
                m_patience);

        log_info() << result << ".";

        tensor4d_t residuals_train(cat_dims(task.size(fold_train), m_odims));
        tensor3d_t residuals_pos1(m_odims), residuals_pos2(m_odims);
        tensor3d_t residuals_neg1(m_odims), residuals_neg2(m_odims);

        stump_t stump;
        tensor4d_t stump_outputs_train(cat_dims(task.size(fold_train), m_odims));

        tensor4d_t targets(cat_dims(task.size(fold_train), m_odims));
        for (size_t i = 0, size = task.size(fold_train); i < size; ++ i)
        {
                const auto target = task.target(fold_train, i);
                targets.tensor(i) = target.tensor();
        }

        gboost_lsearch_function_t func(targets, outputs_train, stump_outputs_train, loss);

        for (auto round = 0; round < m_rounds && !result.is_done(); ++ round)
        {
                for (size_t i = 0, size = task.size(fold_train); i < size; ++ i)
                {
                        const auto input = task.input(fold_train, i);
                        const auto target = task.target(fold_train, i);
                        const auto output = outputs_train.tensor(i);

                        const auto vgrad = loss.vgrad(target, output);
                        residuals_train.vector(i) = -vgrad.vector();
                }

                scalar_t best_value = std::numeric_limits<scalar_t>::max();

                // todo: generalize this - e.g. to use features that are products of two input features
                for (auto feature = 0; feature < nano::size(m_idims); ++ feature)
                {
                        scalars_t fvalues(task.size(fold_train));
                        for (size_t i = 0, size = task.size(fold_train); i < size; ++ i)
                        {
                                const auto input = task.input(fold_train, i);
                                fvalues[i] = input(feature);
                        }

                        auto thresholds = fvalues;
                        std::sort(thresholds.begin(), thresholds.end());
                        thresholds.erase(std::unique(thresholds.begin(), thresholds.end()), thresholds.end());

                        for (size_t t = 0; t + 1 < thresholds.size(); ++ t)
                        {
                                const auto threshold = (thresholds[t + 0] + thresholds[t + 1]) / 2;

                                residuals_pos1.zero(), residuals_pos2.zero();
                                residuals_neg1.zero(), residuals_neg2.zero();

                                int cnt_pos = 0, cnt_neg = 0;
                                for (size_t i = 0, size = task.size(fold_train); i < size; ++ i)
                                {
                                        const auto residual = residuals_train.tensor(i).array();
                                        if (fvalues[i] < threshold)
                                        {
                                                cnt_neg ++;
                                                residuals_neg1.array() += residual;
                                                residuals_neg2.array() += residual * residual;
                                        }
                                        else
                                        {
                                                cnt_pos ++;
                                                residuals_pos1.array() += residual;
                                                residuals_pos2.array() += residual * residual;
                                        }
                                }

                                const auto value =
                                        (residuals_pos2.array().sum() - residuals_pos1.array().square().sum() / cnt_pos) +
                                        (residuals_neg2.array().sum() - residuals_neg1.array().square().sum() / cnt_neg);

                                //log_info() << "feature = " << feature
                                //        << ", threshold = " << threshold
                                //        << ", value = " << value
                                //        << ", count = " << cnt_neg << "+" << cnt_pos << "=" << task.size(fold_train);

                                if (value < best_value)
                                {
                                        best_value = value;
                                        stump.m_feature = feature;
                                        stump.m_threshold = threshold;
                                        stump.m_outputs.resize(cat_dims(2, m_odims));
                                        stump.m_outputs.vector(0) = residuals_neg1.vector() / cnt_neg;
                                        stump.m_outputs.vector(1) = residuals_pos1.vector() / cnt_pos;
                                }

                                // todo: fit both real and discrete stumps
                        }
                }

                // line-search
                for (size_t i = 0, size = task.size(fold_train); i < size; ++ i)
                {
                        const auto input = task.input(fold_train, i);
                        const auto oindex = input(stump.m_feature) < stump.m_threshold ? 0 : 1;
                        stump_outputs_train.tensor(i) = stump.m_outputs.tensor(oindex);
                }

                const auto state = solver->minimize(100, epsilon2<scalar_t>(), func, vector_t::Constant(1, 0));
                const auto step = state.x(0);

                stump.m_outputs.vector() *= step;
                m_stumps.push_back(stump);

                // update current outputs
                update(task, fold_train, outputs_train, stump);
                update(task, fold_valid, outputs_valid, stump);
                update(task, fold_test, outputs_test, stump);

                std::tie(errors_train, values_train) = ::measure(task, fold_train, outputs_train, loss);
                std::tie(errors_valid, values_valid) = ::measure(task, fold_valid, outputs_valid, loss);
                std::tie(errors_test, values_test) = ::measure(task, fold_test, outputs_test, loss);

                result.update(trainer_state_t{timer.milliseconds(), round + 1,
                        {values_train.avg(), errors_train.avg()},
                        {values_valid.avg(), errors_valid.avg()},
                        {values_test.avg(), errors_test.avg()}},
                        m_patience);

                log_info() << result
                        << ",feature=" << stump.m_feature
                        << ",solver=(" << state.m_status
                        << ",i=" << state.m_iterations
                        << ",x=" << state.x(0)
                        << ",f=" << state.f
                        << ",g=" << state.convergence_criteria() << ").";
        }

        // keep only the stumps up to optimum epoch (on the validation dataset)
        m_stumps.erase(
                m_stumps.begin() + result.optimum().m_epoch,
                m_stumps.end());

        return result;
}

tensor3d_t gboost_stump_t::output(const tensor3d_t& input) const
{
        assert(input.dims() == m_idims);

        tensor3d_t output(m_odims);
        output.zero();

        const auto idata = input.array();
        auto odata = output.array();

        for (const auto& stump : m_stumps)
        {
                const auto oindex = idata(stump.m_feature) < stump.m_threshold ? 0 : 1;
                odata.array() += stump.m_outputs.array(oindex);
        }

        return output;
}

bool gboost_stump_t::save(obstream_t& stream) const
{
        if (    !stream.write(m_idims) ||
                !stream.write(m_odims) ||
                !stream.write(m_rounds) ||
                !stream.write(m_stype) ||
                !stream.write(m_rtype) ||
                !stream.write(m_stumps.size()))
        {
                return false;
        }

        for (const auto& stump : m_stumps)
        {
                assert(stump.m_feature >= 0 && stump.m_feature < nano::size(m_idims));
                assert(stump.m_outputs.dims() == cat_dims(2, m_odims));

                if (    !stream.write(stump.m_feature) ||
                        !stream.write(stump.m_threshold) ||
                        !stream.write_tensor(stump.m_outputs))
                {
                       return false;
                }
        }

        return true;
}

bool gboost_stump_t::load(ibstream_t& stream)
{
        size_t n_stumps = 0;
        if (    !stream.read(m_idims) ||
                !stream.read(m_odims) ||
                !stream.read(m_rounds) ||
                !stream.read(m_stype) ||
                !stream.read(m_rtype) ||
                !stream.read(n_stumps))
        {
                return false;
        }

        m_stumps.resize(n_stumps);
        for (auto& stump : m_stumps)
        {
                if (    !stream.read(stump.m_feature) ||
                        !stream.read(stump.m_threshold) ||
                        !stream.read_tensor(stump.m_outputs) ||
                        stump.m_feature < 0 ||
                        stump.m_feature >= nano::size(m_idims) ||
                        stump.m_outputs.dims() != cat_dims(2, m_odims))
                {
                        return false;
                }
        }

        // todo: more verbose loading (#stumps, feature or coefficient statistics, idims...)

        return true;
}

probes_t gboost_stump_t::probes() const
{
        // todo: add probes here to measure the training and the evaluation time
        probes_t probes;
        return probes;
}
<|endoftext|>


<commit_before>#include "Transports/AddressFactory.hpp"
#include "Utils/Logging.hpp"

#include "Settings.hpp"

using Dissent::Utils::Logging;

namespace Dissent {
namespace Applications {
  Settings::Settings(const QString &file, bool actions) :
    _use_file(true),
    _settings(new QSettings(file, QSettings::IniFormat))
  {
    Init(actions);
  }

  Settings::Settings() :
    _use_file(false),
    _settings(new QSettings())
  {
    Init();
  }

  Settings::Settings(const QSharedPointer<QSettings> &settings,
      bool file, bool actions) :
    _use_file(file),
    _settings(settings)
  {
    Init(actions);
  }

  void Settings::Init(bool actions)
  {
    if(_settings->value(Param<Params::Help>(), false).toBool()) {
      Help = true;
      return;
    }
    Help = false;

    QVariant remote = _settings->value(Param<Params::RemoteEndPoints>());
    RemoteEndPoints = ParseAddressList("RemoteEndPoints", remote);

    QVariant local = _settings->value(Param<Params::LocalEndPoints>());
    LocalEndPoints = ParseAddressList("EndPoint", local);

    Auth = _settings->value(Param<Params::Auth>(), true).toBool();
    LocalNodeCount = _settings->value(Param<Params::LocalNodeCount>(), 1).toInt();
    Console = _settings->value(Param<Params::Console>(), false).toBool();
    ExitTunnel = _settings->value(Param<Params::ExitTunnel>(), false).toBool();
    Multithreading = _settings->value(Param<Params::Multithreading>(), false).toBool();

    WebServerUrl = TryParseUrl(_settings->value(Param<Params::WebServerUrl>()).toString(), "http");
    WebServer = WebServerUrl != QUrl();

    EntryTunnelUrl = TryParseUrl(_settings->value(Param<Params::EntryTunnelUrl>()).toString(), "tcp");
    EntryTunnel = EntryTunnelUrl != QUrl();

    ExitTunnelProxyUrl = TryParseUrl(_settings->value(Param<Params::ExitTunnelProxyUrl>()).toString(), "tcp");
    ExitTunnel = (ExitTunnelProxyUrl != QUrl()) || ExitTunnel;

    if(_settings->contains(Param<Params::RoundType>())) {
      QString stype = _settings->value(Param<Params::RoundType>()).toString();
      RoundType = Anonymity::RoundFactory::GetRoundType(stype);
    } else {
      RoundType = Anonymity::RoundFactory::NULL_ROUND;
    }

    Log = _settings->value(Param<Params::Log>(), "null").toString();

    QString log_lower = Log.toLower();
    if(actions) {
      if(log_lower == "stderr") {
        Logging::UseStderr();
      } else if(log_lower == "stdout") {
        Logging::UseStdout();
      } else if(log_lower == "null" || log_lower.isEmpty()) {
        Logging::Disable();
      } else {
        Logging::UseFile(Log);
      }
    }

    if(_settings->contains(Param<Params::LocalId>())) {
      LocalId = ParseIdList(_settings->value(Param<Params::LocalId>()));
    }

    if(_settings->contains(Param<Params::ServerIds>())) {
      ServerIds = ParseIdList(_settings->value(Param<Params::ServerIds>()));
    }

    PublicKeys = _settings->value(Param<Params::PublicKeys>()).toString();
    PrivateKeys = _settings->value(Param<Params::PrivateKeys>()).toString();
  }

  bool Settings::IsValid()
  {
    if(_use_file && (_settings->status() != QSettings::NoError)) {
      _reason = "File error";
      return false;
    }

    if(!LocalEndPoints.count()) {
      _reason = "No local end points";
      return false;
    }

    if(WebServer && (!WebServerUrl.isValid() || WebServerUrl.isEmpty())) {
      _reason = "Invalid WebServerUrl: " + WebServerUrl.toString();
      return false;
    }

    if(EntryTunnel && (!EntryTunnelUrl.isValid() || EntryTunnelUrl.isEmpty())) {
      _reason = "Invalid EntryTunnelUrl: " + EntryTunnelUrl.toString();
      return false;
    }

    if(!ServerIds.count()) {
      _reason = "No server Ids";
      return false;
    }

    if(Auth && (LocalId.count() != LocalNodeCount)) {
      _reason = QString("Insufficient local ids, found %1, expected %2.").
        arg(LocalId.count()).arg(LocalNodeCount);
      return false;
    }

    if(RoundType == Anonymity::RoundFactory::INVALID) {
      _reason = "Invalid round type: " +
        _settings->value(Param<Params::RoundType>()).toString();
      return false;
    }

    return true;
  }

  QString Settings::GetError()
  {
    IsValid();
    return _reason;
  }

  QList<Transports::Address> Settings::ParseAddressList(const QString &name,
      const QVariant &values)
  {
    QList<Transports::Address> list;
    if(values.isNull()) {
      return list;
    }

    QVariantList varlist = values.toList();
    if(!varlist.empty()) {
      foreach(QVariant value, varlist) {
        list.append(Transports::AddressFactory::GetInstance().
            CreateAddress(ParseUrl(name, value)));
      }
    } else {
      list.append(Transports::AddressFactory::GetInstance().
          CreateAddress(ParseUrl(name, values)));
    }

    return list;
  }

  QUrl Settings::ParseUrl(const QString &name, const QVariant &value)
  {
    QUrl url(value.toString());
    if(!url.isValid()) {
      qFatal("Invalid %s: %s", name.toLatin1().data(),
          value.toString().toLatin1().data());
    }
    return url;
  }

  QUrl Settings::TryParseUrl(const QString &string_rep, const QString &scheme)
  {
    QUrl url = QUrl(string_rep);
    if(url.toString() != string_rep) {
      return QUrl();
    }

    if(url.scheme() != scheme) {
      return QUrl();
    }
    return url;
  }

  QList<Connections::Id> Settings::ParseIdList(const QVariant &qids)
  {
    QList<Connections::Id> id_list;

    QVariantList ids = qids.toList();
    if(!ids.empty()) {
      foreach(const QVariant &id, ids) {
        id_list.append(Connections::Id(id.toString()));
      }
    } else {
      id_list.append(Connections::Id(qids.toString()));
    }

    return id_list;
  }

  void Settings::Save()
  {
    if(!_use_file) {
      return;
    }

    QStringList peers;
    foreach(const Transports::Address &addr, RemoteEndPoints) {
      peers << addr.ToString();
    }

    if(!peers.empty()) {
      _settings->setValue(Param<Params::RemoteEndPoints>(), peers);
    }

    QStringList endpoints;
    foreach(const Transports::Address &addr, LocalEndPoints) {
      endpoints << addr.ToString();
    }

    if(!endpoints.empty()) {
      _settings->setValue(Param<Params::LocalEndPoints>(), endpoints);
    }

    _settings->setValue(Param<Params::LocalNodeCount>(), LocalNodeCount);
    _settings->setValue(Param<Params::WebServerUrl>(), WebServerUrl);
    _settings->setValue(Param<Params::Console>(), Console);
    _settings->setValue(Param<Params::Auth>(), Auth);
    _settings->setValue(Param<Params::Log>(), Log);
    _settings->setValue(Param<Params::Multithreading>(), Multithreading);

    QVariantList local_ids;
    foreach(const Connections::Id &id, LocalId) {
      local_ids.append(id.ToString());
    }
    _settings->setValue(Param<Params::LocalId>(), local_ids);

    QVariantList server_ids;
    foreach(const Connections::Id &id, ServerIds) {
      server_ids.append(id.ToString());
    }
    _settings->setValue(Param<Params::ServerIds>(), server_ids);
  }

  Settings Settings::CommandLineParse(const QStringList &params, bool actions)
  {
    QSharedPointer<QxtCommandOptions> options = GetOptions();
    options->parse(params);
    QSharedPointer<QSettings> settings;
    bool file = (options->positional().count() > 0);

    if(file) {
      settings = QSharedPointer<QSettings>(
          new QSettings(options->positional()[0], QSettings::IniFormat));
    } else {
      settings = QSharedPointer<QSettings>(new QSettings());
      // Bug in other platforms?? I do not know...
      settings->clear();
      if(params.size() == 1) {
        settings->setValue(Param<Params::Help>(), true);
      }
    }

    QMultiHash<QString, QVariant> kv_params = options->parameters();

    if(kv_params.value(Param<Params::Help>(), false).toBool() && file) {
      file = false;
      settings = QSharedPointer<QSettings>(new QSettings());
    }

    foreach(const QString &key, kv_params.uniqueKeys()) {
      if(options->value(key).type() == QVariant::String &&
          options->value(key).toString().isEmpty())
      {
        settings->setValue(key, true);
      } else {
        settings->setValue(key, options->value(key));
      }
    }

    return Settings(settings, file, actions);
  }

  QSharedPointer<QxtCommandOptions> Settings::GetOptions()
  {
    QSharedPointer<QxtCommandOptions> options(new QxtCommandOptions());

    options->add(Param<Params::Help>(),
        "help (this screen)",
        QxtCommandOptions::NoValue);

    options->add(Param<Params::RemoteEndPoints>(),
        "list of remote end points",
        QxtCommandOptions::ValueRequired | QxtCommandOptions::AllowMultiple);

    options->add(Param<Params::LocalEndPoints>(),
        "list of local end points",
        QxtCommandOptions::ValueRequired | QxtCommandOptions::AllowMultiple);

    options->add(Param<Params::LocalNodeCount>(),
        "number of virtual nodes to start",
        QxtCommandOptions::ValueRequired);

    options->add(Param<Params::Auth>(),
        "bool, enable or disable authentication",
        QxtCommandOptions::ValueRequired);

    options->add(Param<Params::RoundType>(),
        "the type of round",
        QxtCommandOptions::ValueRequired);

    options->add(Param<Params::Log>(),
        "logging mechanism: stderr, stdout, or a file path",
        QxtCommandOptions::ValueRequired);

    options->add(Param<Params::Console>(),
        "enable console",
        QxtCommandOptions::NoValue);

    options->add(Param<Params::WebServerUrl>(),
        "web server url (enables web server)",
        QxtCommandOptions::ValueRequired);

    options->add(Param<Params::EntryTunnelUrl>(),
        "entry tunnel url (enables entry tunnel)",
        QxtCommandOptions::ValueRequired);

    options->add(Param<Params::ExitTunnel>(),
        "enables exit tunnel",
        QxtCommandOptions::NoValue);

    options->add(Param<Params::ExitTunnelProxyUrl>(),
        "enables redirecting to a proxy at the end of an exit tunnel",
        QxtCommandOptions::ValueRequired);

    options->add(Param<Params::Multithreading>(),
        "enables multithreading",
        QxtCommandOptions::NoValue);

    options->add(Param<Params::LocalId>(),
        "one or more 160-bit base64 local id",
        QxtCommandOptions::ValueRequired | QxtCommandOptions::AllowMultiple);

    options->add(Param<Params::ServerIds>(),
        "one or more 160-bit base64 server id",
        QxtCommandOptions::ValueRequired);

    options->add(Param<Params::PrivateKeys>(),
        "a path to a directory containing private keys",
        QxtCommandOptions::ValueRequired | QxtCommandOptions::AllowMultiple);

    options->add(Param<Params::PublicKeys>(),
        "a path to a directory containing public keys (public keys end in \".pub\"",
        QxtCommandOptions::ValueRequired);

    return options;
  }
}
}
<commit_msg>[Applications] For some reason this has help set to true by default<commit_after>#include "Transports/AddressFactory.hpp"
#include "Utils/Logging.hpp"

#include "Settings.hpp"

using Dissent::Utils::Logging;

namespace Dissent {
namespace Applications {
  Settings::Settings(const QString &file, bool actions) :
    _use_file(true),
    _settings(new QSettings(file, QSettings::IniFormat))
  {
    Init(actions);
  }

  Settings::Settings() :
    _use_file(false),
    _settings(new QSettings())
  {
    _settings->clear();
    Init();
  }

  Settings::Settings(const QSharedPointer<QSettings> &settings,
      bool file, bool actions) :
    _use_file(file),
    _settings(settings)
  {
    Init(actions);
  }

  void Settings::Init(bool actions)
  {
    if(_settings->value(Param<Params::Help>(), false).toBool()) {
      Help = true;
      return;
    }
    Help = false;

    QVariant remote = _settings->value(Param<Params::RemoteEndPoints>());
    RemoteEndPoints = ParseAddressList("RemoteEndPoints", remote);

    QVariant local = _settings->value(Param<Params::LocalEndPoints>());
    LocalEndPoints = ParseAddressList("EndPoint", local);

    Auth = _settings->value(Param<Params::Auth>(), true).toBool();
    LocalNodeCount = _settings->value(Param<Params::LocalNodeCount>(), 1).toInt();
    Console = _settings->value(Param<Params::Console>(), false).toBool();
    ExitTunnel = _settings->value(Param<Params::ExitTunnel>(), false).toBool();
    Multithreading = _settings->value(Param<Params::Multithreading>(), false).toBool();

    WebServerUrl = TryParseUrl(_settings->value(Param<Params::WebServerUrl>()).toString(), "http");
    WebServer = WebServerUrl != QUrl();

    EntryTunnelUrl = TryParseUrl(_settings->value(Param<Params::EntryTunnelUrl>()).toString(), "tcp");
    EntryTunnel = EntryTunnelUrl != QUrl();

    ExitTunnelProxyUrl = TryParseUrl(_settings->value(Param<Params::ExitTunnelProxyUrl>()).toString(), "tcp");
    ExitTunnel = (ExitTunnelProxyUrl != QUrl()) || ExitTunnel;

    if(_settings->contains(Param<Params::RoundType>())) {
      QString stype = _settings->value(Param<Params::RoundType>()).toString();
      RoundType = Anonymity::RoundFactory::GetRoundType(stype);
    } else {
      RoundType = Anonymity::RoundFactory::NULL_ROUND;
    }

    Log = _settings->value(Param<Params::Log>(), "null").toString();

    QString log_lower = Log.toLower();
    if(actions) {
      if(log_lower == "stderr") {
        Logging::UseStderr();
      } else if(log_lower == "stdout") {
        Logging::UseStdout();
      } else if(log_lower == "null" || log_lower.isEmpty()) {
        Logging::Disable();
      } else {
        Logging::UseFile(Log);
      }
    }

    if(_settings->contains(Param<Params::LocalId>())) {
      LocalId = ParseIdList(_settings->value(Param<Params::LocalId>()));
    }

    if(_settings->contains(Param<Params::ServerIds>())) {
      ServerIds = ParseIdList(_settings->value(Param<Params::ServerIds>()));
    }

    PublicKeys = _settings->value(Param<Params::PublicKeys>()).toString();
    PrivateKeys = _settings->value(Param<Params::PrivateKeys>()).toString();
  }

  bool Settings::IsValid()
  {
    if(_use_file && (_settings->status() != QSettings::NoError)) {
      _reason = "File error";
      return false;
    }

    if(!LocalEndPoints.count()) {
      _reason = "No local end points";
      return false;
    }

    if(WebServer && (!WebServerUrl.isValid() || WebServerUrl.isEmpty())) {
      _reason = "Invalid WebServerUrl: " + WebServerUrl.toString();
      return false;
    }

    if(EntryTunnel && (!EntryTunnelUrl.isValid() || EntryTunnelUrl.isEmpty())) {
      _reason = "Invalid EntryTunnelUrl: " + EntryTunnelUrl.toString();
      return false;
    }

    if(!ServerIds.count()) {
      _reason = "No server Ids";
      return false;
    }

    if(Auth && (LocalId.count() != LocalNodeCount)) {
      _reason = QString("Insufficient local ids, found %1, expected %2.").
        arg(LocalId.count()).arg(LocalNodeCount);
      return false;
    }

    if(RoundType == Anonymity::RoundFactory::INVALID) {
      _reason = "Invalid round type: " +
        _settings->value(Param<Params::RoundType>()).toString();
      return false;
    }

    return true;
  }

  QString Settings::GetError()
  {
    IsValid();
    return _reason;
  }

  QList<Transports::Address> Settings::ParseAddressList(const QString &name,
      const QVariant &values)
  {
    QList<Transports::Address> list;
    if(values.isNull()) {
      return list;
    }

    QVariantList varlist = values.toList();
    if(!varlist.empty()) {
      foreach(QVariant value, varlist) {
        list.append(Transports::AddressFactory::GetInstance().
            CreateAddress(ParseUrl(name, value)));
      }
    } else {
      list.append(Transports::AddressFactory::GetInstance().
          CreateAddress(ParseUrl(name, values)));
    }

    return list;
  }

  QUrl Settings::ParseUrl(const QString &name, const QVariant &value)
  {
    QUrl url(value.toString());
    if(!url.isValid()) {
      qFatal("Invalid %s: %s", name.toLatin1().data(),
          value.toString().toLatin1().data());
    }
    return url;
  }

  QUrl Settings::TryParseUrl(const QString &string_rep, const QString &scheme)
  {
    QUrl url = QUrl(string_rep);
    if(url.toString() != string_rep) {
      return QUrl();
    }

    if(url.scheme() != scheme) {
      return QUrl();
    }
    return url;
  }

  QList<Connections::Id> Settings::ParseIdList(const QVariant &qids)
  {
    QList<Connections::Id> id_list;

    QVariantList ids = qids.toList();
    if(!ids.empty()) {
      foreach(const QVariant &id, ids) {
        id_list.append(Connections::Id(id.toString()));
      }
    } else {
      id_list.append(Connections::Id(qids.toString()));
    }

    return id_list;
  }

  void Settings::Save()
  {
    if(!_use_file) {
      return;
    }

    QStringList peers;
    foreach(const Transports::Address &addr, RemoteEndPoints) {
      peers << addr.ToString();
    }

    if(!peers.empty()) {
      _settings->setValue(Param<Params::RemoteEndPoints>(), peers);
    }

    QStringList endpoints;
    foreach(const Transports::Address &addr, LocalEndPoints) {
      endpoints << addr.ToString();
    }

    if(!endpoints.empty()) {
      _settings->setValue(Param<Params::LocalEndPoints>(), endpoints);
    }

    _settings->setValue(Param<Params::LocalNodeCount>(), LocalNodeCount);
    _settings->setValue(Param<Params::WebServerUrl>(), WebServerUrl);
    _settings->setValue(Param<Params::Console>(), Console);
    _settings->setValue(Param<Params::Auth>(), Auth);
    _settings->setValue(Param<Params::Log>(), Log);
    _settings->setValue(Param<Params::Multithreading>(), Multithreading);

    QVariantList local_ids;
    foreach(const Connections::Id &id, LocalId) {
      local_ids.append(id.ToString());
    }
    _settings->setValue(Param<Params::LocalId>(), local_ids);

    QVariantList server_ids;
    foreach(const Connections::Id &id, ServerIds) {
      server_ids.append(id.ToString());
    }
    _settings->setValue(Param<Params::ServerIds>(), server_ids);
  }

  Settings Settings::CommandLineParse(const QStringList &params, bool actions)
  {
    QSharedPointer<QxtCommandOptions> options = GetOptions();
    options->parse(params);
    QSharedPointer<QSettings> settings;
    bool file = (options->positional().count() > 0);

    if(file) {
      settings = QSharedPointer<QSettings>(
          new QSettings(options->positional()[0], QSettings::IniFormat));
    } else {
      settings = QSharedPointer<QSettings>(new QSettings());
      // Bug in other platforms?? I do not know...
      settings->clear();
      if(params.size() == 1) {
        settings->setValue(Param<Params::Help>(), true);
      }
    }

    QMultiHash<QString, QVariant> kv_params = options->parameters();

    if(kv_params.value(Param<Params::Help>(), false).toBool() && file) {
      file = false;
      settings = QSharedPointer<QSettings>(new QSettings());
    }

    foreach(const QString &key, kv_params.uniqueKeys()) {
      if(options->value(key).type() == QVariant::String &&
          options->value(key).toString().isEmpty())
      {
        settings->setValue(key, true);
      } else {
        settings->setValue(key, options->value(key));
      }
    }

    return Settings(settings, file, actions);
  }

  QSharedPointer<QxtCommandOptions> Settings::GetOptions()
  {
    QSharedPointer<QxtCommandOptions> options(new QxtCommandOptions());

    options->add(Param<Params::Help>(),
        "help (this screen)",
        QxtCommandOptions::NoValue);

    options->add(Param<Params::RemoteEndPoints>(),
        "list of remote end points",
        QxtCommandOptions::ValueRequired | QxtCommandOptions::AllowMultiple);

    options->add(Param<Params::LocalEndPoints>(),
        "list of local end points",
        QxtCommandOptions::ValueRequired | QxtCommandOptions::AllowMultiple);

    options->add(Param<Params::LocalNodeCount>(),
        "number of virtual nodes to start",
        QxtCommandOptions::ValueRequired);

    options->add(Param<Params::Auth>(),
        "bool, enable or disable authentication",
        QxtCommandOptions::ValueRequired);

    options->add(Param<Params::RoundType>(),
        "the type of round",
        QxtCommandOptions::ValueRequired);

    options->add(Param<Params::Log>(),
        "logging mechanism: stderr, stdout, or a file path",
        QxtCommandOptions::ValueRequired);

    options->add(Param<Params::Console>(),
        "enable console",
        QxtCommandOptions::NoValue);

    options->add(Param<Params::WebServerUrl>(),
        "web server url (enables web server)",
        QxtCommandOptions::ValueRequired);

    options->add(Param<Params::EntryTunnelUrl>(),
        "entry tunnel url (enables entry tunnel)",
        QxtCommandOptions::ValueRequired);

    options->add(Param<Params::ExitTunnel>(),
        "enables exit tunnel",
        QxtCommandOptions::NoValue);

    options->add(Param<Params::ExitTunnelProxyUrl>(),
        "enables redirecting to a proxy at the end of an exit tunnel",
        QxtCommandOptions::ValueRequired);

    options->add(Param<Params::Multithreading>(),
        "enables multithreading",
        QxtCommandOptions::NoValue);

    options->add(Param<Params::LocalId>(),
        "one or more 160-bit base64 local id",
        QxtCommandOptions::ValueRequired | QxtCommandOptions::AllowMultiple);

    options->add(Param<Params::ServerIds>(),
        "one or more 160-bit base64 server id",
        QxtCommandOptions::ValueRequired);

    options->add(Param<Params::PrivateKeys>(),
        "a path to a directory containing private keys",
        QxtCommandOptions::ValueRequired | QxtCommandOptions::AllowMultiple);

    options->add(Param<Params::PublicKeys>(),
        "a path to a directory containing public keys (public keys end in \".pub\"",
        QxtCommandOptions::ValueRequired);

    return options;
  }
}
}
<|endoftext|>