<commit_before>// Copyright 2010-2013 RethinkDB, all rights reserved.
#ifndef CONTAINERS_SCOPED_HPP_
#define CONTAINERS_SCOPED_HPP_

#include <string.h>

#include <utility>

#include "errors.hpp"
#include "utils.hpp"

// Like boost::scoped_ptr only with release, init, no bool conversion, no boost headers!
template <class T>
class scoped_ptr_t {
public:
    bool operator<(const scoped_ptr_t &other) const {
        return *ptr_ < *other;
    }

    template <class U>
    friend class scoped_ptr_t;

    scoped_ptr_t() : ptr_(NULL) { }
    explicit scoped_ptr_t(T *p) : ptr_(p) { }

    // (These noexcepts don't actually do anything w.r.t. STL containers, since the
    // type's not copyable.  There is no specific reason why these are many other
    // functions need be marked noexcept with any degree of urgency.)
    scoped_ptr_t(scoped_ptr_t &&movee) noexcept : ptr_(movee.ptr_) {
        movee.ptr_ = NULL;
    }
    template <class U>
    scoped_ptr_t(scoped_ptr_t<U> &&movee) noexcept : ptr_(movee.ptr_) {
        movee.ptr_ = NULL;
    }

    ~scoped_ptr_t() {
        reset();
    }

    scoped_ptr_t &operator=(scoped_ptr_t &&movee) noexcept {
        scoped_ptr_t tmp(std::move(movee));
        swap(tmp);
        return *this;
    }

    template <class U>
    scoped_ptr_t &operator=(scoped_ptr_t<U> &&movee) noexcept {
        scoped_ptr_t tmp(std::move(movee));
        swap(tmp);
        return *this;
    }

    // These 'init' functions are largely obsolete, because move semantics are a
    // better thing to use.
    template <class U>
    void init(scoped_ptr_t<U> &&movee) {
        rassert(ptr_ == NULL);

        operator=(std::move(movee));
    }

    // includes a sanity-check for first-time use.
    void init(T *value) {
        rassert(ptr_ == NULL);

        // This is like reset with an assert.
        T *tmp = ptr_;
        ptr_ = value;
        delete tmp;
    }

    void reset() {
        T *tmp = ptr_;
        ptr_ = NULL;
        delete tmp;
    }

    MUST_USE T *release() {
        T *tmp = ptr_;
        ptr_ = NULL;
        return tmp;
    }

    void swap(scoped_ptr_t &other) noexcept {
        T *tmp = ptr_;
        ptr_ = other.ptr_;
        other.ptr_ = tmp;
    }

    T &operator*() const {
        rassert(ptr_);
        return *ptr_;
    }

    T *get() const {
        rassert(ptr_);
        return ptr_;
    }

    T *get_or_null() const {
        return ptr_;
    }

    T *operator->() const {
        rassert(ptr_);
        return ptr_;
    }

    bool has() const {
        return ptr_ != NULL;
    }

private:
    T *ptr_;

    DISABLE_COPYING(scoped_ptr_t);
};

template <class T, class... Args>
scoped_ptr_t<T> make_scoped(Args&&... args) {
    return scoped_ptr_t<T>(new T(std::forward<Args>(args)...));
}

// Not really like boost::scoped_array.  A fascist array.
template <class T>
class scoped_array_t {
public:
    scoped_array_t() : ptr_(NULL), size_(0) { }
    explicit scoped_array_t(size_t n) : ptr_(NULL), size_(0) {
        init(n);
    }

    scoped_array_t(T *ptr, size_t size) : ptr_(NULL), size_(0) {
        init(ptr, size);
    }

    // (These noexcepts don't actually do anything w.r.t. STL containers, since the
    // type's not copyable.  There is no specific reason why these are many other
    // functions need be marked noexcept with any degree of urgency.)
    scoped_array_t(scoped_array_t &&movee) noexcept
        : ptr_(movee.ptr_), size_(movee.size_) {
        movee.ptr_ = NULL;
        movee.size_ = 0;
    }

    ~scoped_array_t() {
        reset();
    }

    scoped_array_t &operator=(scoped_array_t &&movee) noexcept {
        scoped_array_t tmp(std::move(movee));
        swap(tmp);
        return *this;
    }

    void init(size_t n) {
        rassert(ptr_ == NULL);
        ptr_ = new T[n];
        size_ = n;
    }

    // The opposite of release.
    void init(T *ptr, size_t size) {
        rassert(ptr != NULL);
        rassert(ptr_ == NULL);

        ptr_ = ptr;
        size_ = size;
    }

    void reset() {
        T *tmp = ptr_;
        ptr_ = NULL;
        size_ = 0;
        delete[] tmp;
    }

    MUST_USE T *release(size_t *size_out) {
        *size_out = size_;
        T *tmp = ptr_;
        ptr_ = NULL;
        size_ = 0;
        return tmp;
    }

    void swap(scoped_array_t &other) noexcept {
        T *tmp = ptr_;
        size_t tmpsize = size_;
        ptr_ = other.ptr_;
        size_ = other.size_;
        other.ptr_ = tmp;
        other.size_ = tmpsize;
    }

    T &operator[](size_t i) const {
        rassert(ptr_);
        rassert(i < size_);
        return ptr_[i];
    }

    T *data() const {
        rassert(ptr_);
        return ptr_;
    }

    size_t size() const {
        rassert(ptr_);
        return size_;
    }

    bool has() const {
        return ptr_ != NULL;
    }

private:
    T *ptr_;
    size_t size_;

    DISABLE_COPYING(scoped_array_t);
};

// For dumb structs that get rmalloc/free for allocation.

template <class T>
class scoped_malloc_t {
public:
    template <class U>
    friend class scoped_malloc_t;

    scoped_malloc_t() : ptr_(NULL) { }
    explicit scoped_malloc_t(void *ptr) : ptr_(static_cast<T *>(ptr)) { }
    explicit scoped_malloc_t(size_t n) : ptr_(static_cast<T *>(rmalloc(n))) { }
    scoped_malloc_t(const char *beg, const char *end) {
        rassert(beg <= end);
        size_t n = end - beg;
        ptr_ = static_cast<T *>(rmalloc(n));
        memcpy(ptr_, beg, n);
    }
    // (These noexcepts don't actually do anything w.r.t. STL containers, since the
    // type's not copyable.  There is no specific reason why these are many other
    // functions need be marked noexcept with any degree of urgency.)
    scoped_malloc_t(scoped_malloc_t &&movee) noexcept
        : ptr_(movee.ptr_) {
        movee.ptr_ = NULL;
    }

    template <class U>
    scoped_malloc_t(scoped_malloc_t<U> &&movee) noexcept
        : ptr_(movee.ptr_) {
        movee.ptr_ = NULL;
    }

    ~scoped_malloc_t() {
        free(ptr_);
    }

    void operator=(scoped_malloc_t &&movee) noexcept {
        scoped_malloc_t tmp(std::move(movee));
        swap(tmp);
    }

    void init(void *ptr) {
        guarantee(ptr_ == NULL);
        ptr_ = static_cast<T *>(ptr);
    }

    T *get() const { return ptr_; }
    T *operator->() const { return ptr_; }

    T *release() {
        T *tmp = ptr_;
        ptr_ = NULL;
        return tmp;
    }

    void reset() {
        scoped_malloc_t tmp;
        swap(tmp);
    }

    bool has() const {
        return ptr_ != NULL;
    }

private:
    void swap(scoped_malloc_t &other) noexcept {  // NOLINT
        T *tmp = ptr_;
        ptr_ = other.ptr_;
        other.ptr_ = tmp;
    }

    T *ptr_;

    DISABLE_COPYING(scoped_malloc_t);
};

#endif  // CONTAINERS_SCOPED_HPP_
<commit_msg>Removed the thankfully unused scoped_ptr_t::operator<.<commit_after>// Copyright 2010-2013 RethinkDB, all rights reserved.
#ifndef CONTAINERS_SCOPED_HPP_
#define CONTAINERS_SCOPED_HPP_

#include <string.h>

#include <utility>

#include "errors.hpp"
#include "utils.hpp"

// Like boost::scoped_ptr only with release, init, no bool conversion, no boost headers!
template <class T>
class scoped_ptr_t {
public:
    template <class U>
    friend class scoped_ptr_t;

    scoped_ptr_t() : ptr_(NULL) { }
    explicit scoped_ptr_t(T *p) : ptr_(p) { }

    // (These noexcepts don't actually do anything w.r.t. STL containers, since the
    // type's not copyable.  There is no specific reason why these are many other
    // functions need be marked noexcept with any degree of urgency.)
    scoped_ptr_t(scoped_ptr_t &&movee) noexcept : ptr_(movee.ptr_) {
        movee.ptr_ = NULL;
    }
    template <class U>
    scoped_ptr_t(scoped_ptr_t<U> &&movee) noexcept : ptr_(movee.ptr_) {
        movee.ptr_ = NULL;
    }

    ~scoped_ptr_t() {
        reset();
    }

    scoped_ptr_t &operator=(scoped_ptr_t &&movee) noexcept {
        scoped_ptr_t tmp(std::move(movee));
        swap(tmp);
        return *this;
    }

    template <class U>
    scoped_ptr_t &operator=(scoped_ptr_t<U> &&movee) noexcept {
        scoped_ptr_t tmp(std::move(movee));
        swap(tmp);
        return *this;
    }

    // These 'init' functions are largely obsolete, because move semantics are a
    // better thing to use.
    template <class U>
    void init(scoped_ptr_t<U> &&movee) {
        rassert(ptr_ == NULL);

        operator=(std::move(movee));
    }

    // includes a sanity-check for first-time use.
    void init(T *value) {
        rassert(ptr_ == NULL);

        // This is like reset with an assert.
        T *tmp = ptr_;
        ptr_ = value;
        delete tmp;
    }

    void reset() {
        T *tmp = ptr_;
        ptr_ = NULL;
        delete tmp;
    }

    MUST_USE T *release() {
        T *tmp = ptr_;
        ptr_ = NULL;
        return tmp;
    }

    void swap(scoped_ptr_t &other) noexcept {
        T *tmp = ptr_;
        ptr_ = other.ptr_;
        other.ptr_ = tmp;
    }

    T &operator*() const {
        rassert(ptr_);
        return *ptr_;
    }

    T *get() const {
        rassert(ptr_);
        return ptr_;
    }

    T *get_or_null() const {
        return ptr_;
    }

    T *operator->() const {
        rassert(ptr_);
        return ptr_;
    }

    bool has() const {
        return ptr_ != NULL;
    }

private:
    T *ptr_;

    DISABLE_COPYING(scoped_ptr_t);
};

template <class T, class... Args>
scoped_ptr_t<T> make_scoped(Args&&... args) {
    return scoped_ptr_t<T>(new T(std::forward<Args>(args)...));
}

// Not really like boost::scoped_array.  A fascist array.
template <class T>
class scoped_array_t {
public:
    scoped_array_t() : ptr_(NULL), size_(0) { }
    explicit scoped_array_t(size_t n) : ptr_(NULL), size_(0) {
        init(n);
    }

    scoped_array_t(T *ptr, size_t size) : ptr_(NULL), size_(0) {
        init(ptr, size);
    }

    // (These noexcepts don't actually do anything w.r.t. STL containers, since the
    // type's not copyable.  There is no specific reason why these are many other
    // functions need be marked noexcept with any degree of urgency.)
    scoped_array_t(scoped_array_t &&movee) noexcept
        : ptr_(movee.ptr_), size_(movee.size_) {
        movee.ptr_ = NULL;
        movee.size_ = 0;
    }

    ~scoped_array_t() {
        reset();
    }

    scoped_array_t &operator=(scoped_array_t &&movee) noexcept {
        scoped_array_t tmp(std::move(movee));
        swap(tmp);
        return *this;
    }

    void init(size_t n) {
        rassert(ptr_ == NULL);
        ptr_ = new T[n];
        size_ = n;
    }

    // The opposite of release.
    void init(T *ptr, size_t size) {
        rassert(ptr != NULL);
        rassert(ptr_ == NULL);

        ptr_ = ptr;
        size_ = size;
    }

    void reset() {
        T *tmp = ptr_;
        ptr_ = NULL;
        size_ = 0;
        delete[] tmp;
    }

    MUST_USE T *release(size_t *size_out) {
        *size_out = size_;
        T *tmp = ptr_;
        ptr_ = NULL;
        size_ = 0;
        return tmp;
    }

    void swap(scoped_array_t &other) noexcept {
        T *tmp = ptr_;
        size_t tmpsize = size_;
        ptr_ = other.ptr_;
        size_ = other.size_;
        other.ptr_ = tmp;
        other.size_ = tmpsize;
    }

    T &operator[](size_t i) const {
        rassert(ptr_);
        rassert(i < size_);
        return ptr_[i];
    }

    T *data() const {
        rassert(ptr_);
        return ptr_;
    }

    size_t size() const {
        rassert(ptr_);
        return size_;
    }

    bool has() const {
        return ptr_ != NULL;
    }

private:
    T *ptr_;
    size_t size_;

    DISABLE_COPYING(scoped_array_t);
};

// For dumb structs that get rmalloc/free for allocation.

template <class T>
class scoped_malloc_t {
public:
    template <class U>
    friend class scoped_malloc_t;

    scoped_malloc_t() : ptr_(NULL) { }
    explicit scoped_malloc_t(void *ptr) : ptr_(static_cast<T *>(ptr)) { }
    explicit scoped_malloc_t(size_t n) : ptr_(static_cast<T *>(rmalloc(n))) { }
    scoped_malloc_t(const char *beg, const char *end) {
        rassert(beg <= end);
        size_t n = end - beg;
        ptr_ = static_cast<T *>(rmalloc(n));
        memcpy(ptr_, beg, n);
    }
    // (These noexcepts don't actually do anything w.r.t. STL containers, since the
    // type's not copyable.  There is no specific reason why these are many other
    // functions need be marked noexcept with any degree of urgency.)
    scoped_malloc_t(scoped_malloc_t &&movee) noexcept
        : ptr_(movee.ptr_) {
        movee.ptr_ = NULL;
    }

    template <class U>
    scoped_malloc_t(scoped_malloc_t<U> &&movee) noexcept
        : ptr_(movee.ptr_) {
        movee.ptr_ = NULL;
    }

    ~scoped_malloc_t() {
        free(ptr_);
    }

    void operator=(scoped_malloc_t &&movee) noexcept {
        scoped_malloc_t tmp(std::move(movee));
        swap(tmp);
    }

    void init(void *ptr) {
        guarantee(ptr_ == NULL);
        ptr_ = static_cast<T *>(ptr);
    }

    T *get() const { return ptr_; }
    T *operator->() const { return ptr_; }

    T *release() {
        T *tmp = ptr_;
        ptr_ = NULL;
        return tmp;
    }

    void reset() {
        scoped_malloc_t tmp;
        swap(tmp);
    }

    bool has() const {
        return ptr_ != NULL;
    }

private:
    void swap(scoped_malloc_t &other) noexcept {  // NOLINT
        T *tmp = ptr_;
        ptr_ = other.ptr_;
        other.ptr_ = tmp;
    }

    T *ptr_;

    DISABLE_COPYING(scoped_malloc_t);
};

#endif  // CONTAINERS_SCOPED_HPP_
<|endoftext|>


<commit_before>#include <fstream>
#include <string>
#include <regex>
#include <vector>
#include <iostream>
#include <boost/program_options.hpp>
#include <boost/program_options/parsers.hpp>
#include "macros.hpp"
#include "ConfigParser.hpp"
#include "FileUtils.hpp"

namespace po = boost::program_options;
using std::string;
using std::vector;
using std::cout;
using std::endl;

/**
 * Check if a given ZMQ endpoint is a valid TCP or IPC endpoint
 */
static bool COLD checkTCPIPCEndpoint(const string& endpoint, bool allowIPC = true) {
    //Check if the parameter is valid
    bool isTCP = endpoint.find("tcp://") == 0;
    bool isIPC = endpoint.find("ipc://") == 0 && allowIPC;
    if(!isTCP && !isIPC) {
        cout << "Endpoint " << endpoint << " is not a valid TCP" << (allowIPC ? " or IPC": "") << " endpoint!" << endl;
        return false;
    }
    return true;
}

void COLD ConfigParser::saveConfigFile() {
    std::ofstream fout("yak.cfg");
    if(!logFile.empty()) {
        fout << "logfile=" << logFile << '\n';
    }
    for(string endpoint : repEndpoints) {
        fout << "rep-endpoint=" << endpoint << '\n';
    }
    for(string endpoint : pullEndpoints) {
        fout << "pull-endpoint=" << endpoint << '\n';
    }
    for(string endpoint : subEndpoints) {
        fout << "sub-endpoint=" << endpoint << '\n';
    }
    if(ipv4Only) {
        fout << "ipv4-only" << '\n';
    }
    fout << "lru-cache-size=" << defaultLRUCacheSize << '\n';
    fout << "table-block-size=" << defaultTableBlockSize << '\n';
    fout << "write-buffer-size=" << defaultWriteBufferSize << '\n';
    fout << "bloom-filter-bits-per-key=" << defaultBloomFilterBitsPerKey << '\n';
    fout << "internal-hwm=" << internalHWM << '\n';
    fout << "external-hwm=" << externalHWM << '\n';
    if(!compressionEnabledPerDefault) {
        fout << "disable-compression=true" << '\n';
    }
    fout.close();
}

COLD ConfigParser::ConfigParser(int argc, char** argv) {
    /**
     * Remember when adding options, saveConfigFile() must also be updated!
     */
    // Declare the supported options.
    string configFileName;
    repEndpoints = {"tcp://*:7100","ipc:///tmp/yakserver-rep"};
    pullEndpoints = {"tcp://*:7101","ipc:///tmp/yakserver-pull"};
    subEndpoints = {"tcp://*:7102","ipc:///tmp/yakserver-sub"};
    httpEndpoint = "tcp://*:7109";
    po::options_description generalOptions("General options");
    generalOptions.add_options()
        ("help,h", "Print help message")
        ("logfile,l", po::value<string>(&logFile)->default_value(""), "The file the log will be written to")
        ("config,c",
            po::value<string>(&configFileName)->default_value("yak.cfg"),
            "The configuration file to use");
    po::options_description socketOptions("Socket options");
    socketOptions.add_options()
        ("req-endpoints,r",
            po::value<vector<string> >(&repEndpoints),
            "The endpoints the REP backend will bind to.\nDefaults to tcp://*:7100, ipc:///tmp/yakserver-rep")
        ("pull-endpoint,p", po::value<vector<string> >(&pullEndpoints),
            "The endpoints the PULL backend will bind to.\nDefaults to tcp://*:7101, ipc:///tmp/yakserver-pull")
        ("sub-endpoint,s", po::value<vector<string> >(&pullEndpoints),
            "The endpoints the SUB backend will bind to.\nDefaults to tcp://*:7102, ipc:///tmp/yakserver-sub")
        ("http-endpoint,e", po::value<vector<string> >(&pullEndpoints),
            "The endpoint the internal HTTP server will listen on. Defaults to tcp://*:7109")
        ("ipv4-only,4","By default the application uses IPv6 sockets to bind to both IPv6 and IPv4. This option tells the application not to use IPv6 capable sockets.")
        ("external-hwm",
            po::value<int>(&externalHWM)->default_value(250),
            "Set the ZMQ High-watermark for external sockets")
        ("internal-hwm",
            po::value<int>(&internalHWM)->default_value(250),
            "Set the ZMQ High-watermark for internal sockets")
    ;
    po::options_description tableOptions("Table options");
    tableOptions.add_options()
        ("lru-cache-size",
            po::value<uint64_t>(&defaultLRUCacheSize)->default_value(1024 * 1024 * 16),
            "Set the default LRU cache size in bytes. Overriden by table-specific options.")
        ("table-block-size",
            po::value<uint64_t>(&defaultTableBlockSize)->default_value(256*1024),
            "Set the default table block size in bytes. Overriden by table-specific options.")
        ("write-buffer-size",
            po::value<uint64_t>(&defaultWriteBufferSize)->default_value(1024 * 1024 * 64),
            "Set the default write buffer size in bytes. Overriden by table-specific options.")
        ("bloom-filter-bits-per-key",
            po::value<uint64_t>(&defaultBloomFilterBitsPerKey)->default_value(0),
            "Set the default bits per key for the bloom filter. Set to 0 to disable bloom filter. Overriden by table-specific options.")
        ("disable-compression,d","By default table compression is enabled for all unconfigured tables. If this option is used, table compression is disabled by default. Overridden by table-specific options.")
    ;
    //Create the main options group
    po::options_description desc("Options");
    desc.add(generalOptions).add(socketOptions).add(tableOptions);

    po::variables_map vm;
    po::store(po::parse_command_line(argc, argv, desc), vm);
    po::notify(vm);
    //Parse the config file
    bool processedConfigFile = false;
    if(fexists(configFileName)) {
        processedConfigFile = true;
        po::store(po::parse_config_file<char>(configFileName.c_str(), desc, true), vm);
    }
    po::notify(vm);
    //Check if --help is given
    if (vm.count("help")) {
        cout << desc << endl;
        exit(1);
    }
    //Check the endpoints
    for(string endpoint : repEndpoints) {
        if(!checkTCPIPCEndpoint(endpoint)) {
            cout << desc << endl;
            exit(1);
        }
    }
    for(string endpoint : pullEndpoints) {
        if(!checkTCPIPCEndpoint(endpoint)) {
            cout << desc << endl;
            exit(1);
        }
    }
    for(string endpoint : subEndpoints) {
        if(!checkTCPIPCEndpoint(endpoint)) {
            cout << desc << endl;
            exit(1);
        }
    }
    if(!checkTCPIPCEndpoint(httpEndpoint, false)) {
        cout << desc << endl;
        exit(1);
    }
    //Get bool-ish options
    this->ipv4Only = (vm.count("ipv4-only") > 0);
    this->compressionEnabledPerDefault = (vm.count("disable-compression") > 0);
    //Write the config data to the config file unless there are no arguments
    if(argc > 1 || (processedConfigFile && configFileName != "yak.cfg")) {
        saveConfigFile();
    }
}

const std::string& ConfigParser::getLogFile() {
    return logFile;
}

const std::vector<std::string>& ConfigParser::getREPEndpoints() {
    return repEndpoints;
}

const std::vector<std::string>& ConfigParser::getPULLEndpoints() {
    return pullEndpoints;
}

const std::vector<std::string>& ConfigParser::getSUBEndpoints() {
    return subEndpoints;
}

const bool ConfigParser::isIPv4Only() {
    return ipv4Only;
}

const std::string& ConfigParser::getHTTPEndpoint() {
    return httpEndpoint;
}

uint64_t ConfigParser::getDefaultLRUCacheSize() {
    return defaultLRUCacheSize;
}

uint64_t ConfigParser::getDefaultTableBlockSize() {
    return defaultTableBlockSize;
}

uint64_t ConfigParser::getDefaultWriteBufferSize() {
    return defaultWriteBufferSize;
}

uint64_t ConfigParser::getDefaultBloomFilterBitsPerKey() {
    return defaultBloomFilterBitsPerKey;
}

bool ConfigParser::isCompressionEnabledPerDefault() {
    return compressionEnabledPerDefault;
}

int ConfigParser::getInternalHWM() {
    return internalHWM;
}

int ConfigParser::getExternalHWM() {
    return externalHWM;
}<commit_msg>Fix config reload issues<commit_after>#include <fstream>
#include <string>
#include <regex>
#include <vector>
#include <iostream>
#include <boost/program_options.hpp>
#include <boost/program_options/parsers.hpp>
#include "macros.hpp"
#include "ConfigParser.hpp"
#include "FileUtils.hpp"

namespace po = boost::program_options;
using std::string;
using std::vector;
using std::cout;
using std::endl;

/**
 * Check if a given ZMQ endpoint is a valid TCP or IPC endpoint
 */
static bool COLD checkTCPIPCEndpoint(const string& endpoint, bool allowIPC = true) {
    //Check if the parameter is valid
    bool isTCP = endpoint.find("tcp://") == 0;
    bool isIPC = endpoint.find("ipc://") == 0 && allowIPC;
    if(!isTCP && !isIPC) {
        cout << "Endpoint " << endpoint << " is not a valid TCP" << (allowIPC ? " or IPC": "") << " endpoint!" << endl;
        return false;
    }
    return true;
}

void COLD ConfigParser::saveConfigFile() {
    std::ofstream fout("yak.cfg");
    if(!logFile.empty()) {
        fout << "logfile=" << logFile << '\n';
    }
    for(string endpoint : repEndpoints) {
        fout << "rep-endpoint=" << endpoint << '\n';
    }
    for(string endpoint : pullEndpoints) {
        fout << "pull-endpoint=" << endpoint << '\n';
    }
    for(string endpoint : subEndpoints) {
        fout << "sub-endpoint=" << endpoint << '\n';
    }
    fout << "http-endpoint=" << httpEndpoint << '\n';
    if(ipv4Only) {
        fout << "ipv4-only" << '\n';
    }
    fout << "lru-cache-size=" << defaultLRUCacheSize << '\n';
    fout << "table-block-size=" << defaultTableBlockSize << '\n';
    fout << "write-buffer-size=" << defaultWriteBufferSize << '\n';
    fout << "bloom-filter-bits-per-key=" << defaultBloomFilterBitsPerKey << '\n';
    fout << "internal-hwm=" << internalHWM << '\n';
    fout << "external-hwm=" << externalHWM << '\n';
    if(!compressionEnabledPerDefault) {
        fout << "disable-compression=true" << '\n';
    }
    fout.close();
}

COLD ConfigParser::ConfigParser(int argc, char** argv) {
    /**
     * Remember when adding options, saveConfigFile() must also be updated!
     */
    // Declare the supported options.
    string configFileName;
    repEndpoints = {"tcp://*:7100","ipc:///tmp/yakserver-rep"};
    pullEndpoints = {"tcp://*:7101","ipc:///tmp/yakserver-pull"};
    subEndpoints = {"tcp://*:7102","ipc:///tmp/yakserver-sub"};
    httpEndpoint = "tcp://*:7109";
    po::options_description generalOptions("General options");
    generalOptions.add_options()
        ("help,h", "Print help message")
        ("logfile,l", po::value<string>(&logFile)->default_value(""), "The file the log will be written to")
        ("config,c",
            po::value<string>(&configFileName)->default_value("yak.cfg"),
            "The configuration file to use");
    po::options_description socketOptions("Socket options");
    socketOptions.add_options()
        ("req-endpoints,r",
            po::value<vector<string> >(&repEndpoints),
            "The endpoints the REP backend will bind to.\nDefaults to tcp://*:7100, ipc:///tmp/yakserver-rep")
        ("pull-endpoint,p", po::value<vector<string> >(&pullEndpoints),
            "The endpoints the PULL backend will bind to.\nDefaults to tcp://*:7101, ipc:///tmp/yakserver-pull")
        ("sub-endpoint,s", po::value<vector<string> >(&subEndpoints),
            "The endpoints the SUB backend will bind to.\nDefaults to tcp://*:7102, ipc:///tmp/yakserver-sub")
        ("http-endpoint,e", po::value<string>(&httpEndpoint),
            "The endpoint the internal HTTP server will listen on. Defaults to tcp://*:7109")
        ("ipv4-only,4","By default the application uses IPv6 sockets to bind to both IPv6 and IPv4. This option tells the application not to use IPv6 capable sockets.")
        ("external-hwm",
            po::value<int>(&externalHWM)->default_value(250),
            "Set the ZMQ High-watermark for external sockets")
        ("internal-hwm",
            po::value<int>(&internalHWM)->default_value(250),
            "Set the ZMQ High-watermark for internal sockets")
    ;
    po::options_description tableOptions("Table options");
    tableOptions.add_options()
        ("lru-cache-size",
            po::value<uint64_t>(&defaultLRUCacheSize)->default_value(1024 * 1024 * 16),
            "Set the default LRU cache size in bytes. Overriden by table-specific options.")
        ("table-block-size",
            po::value<uint64_t>(&defaultTableBlockSize)->default_value(256*1024),
            "Set the default table block size in bytes. Overriden by table-specific options.")
        ("write-buffer-size",
            po::value<uint64_t>(&defaultWriteBufferSize)->default_value(1024 * 1024 * 64),
            "Set the default write buffer size in bytes. Overriden by table-specific options.")
        ("bloom-filter-bits-per-key",
            po::value<uint64_t>(&defaultBloomFilterBitsPerKey)->default_value(0),
            "Set the default bits per key for the bloom filter. Set to 0 to disable bloom filter. Overriden by table-specific options.")
        ("disable-compression,d","By default table compression is enabled for all unconfigured tables. If this option is used, table compression is disabled by default. Overridden by table-specific options.")
    ;
    //Create the main options group
    po::options_description desc("Options");
    desc.add(generalOptions).add(socketOptions).add(tableOptions);

    po::variables_map vm;
    po::store(po::parse_command_line(argc, argv, desc), vm);
    po::notify(vm);
    //Parse the config file
    bool processedConfigFile = false;
    if(fexists(configFileName)) {
        processedConfigFile = true;
        po::store(po::parse_config_file<char>(configFileName.c_str(), desc, true), vm);
    }
    po::notify(vm);
    //Check if --help is given
    if (vm.count("help")) {
        cout << desc << endl;
        exit(1);
    }
    //Check the endpoints
    for(string endpoint : repEndpoints) {
        if(!checkTCPIPCEndpoint(endpoint)) {
            cout << desc << endl;
            exit(1);
        }
    }
    for(string endpoint : pullEndpoints) {
        if(!checkTCPIPCEndpoint(endpoint)) {
            cout << desc << endl;
            exit(1);
        }
    }
    for(string endpoint : subEndpoints) {
        if(!checkTCPIPCEndpoint(endpoint)) {
            cout << desc << endl;
            exit(1);
        }
    }
    if(!checkTCPIPCEndpoint(httpEndpoint, false)) {
        cout << desc << endl;
        exit(1);
    }
    //Get bool-ish options
    this->ipv4Only = (vm.count("ipv4-only") > 0);
    this->compressionEnabledPerDefault = (vm.count("disable-compression") > 0);
    //Write the config data to the config file unless there are no arguments
    if(argc > 1 || (processedConfigFile && configFileName != "yak.cfg")) {
        saveConfigFile();
    }
}

const std::string& ConfigParser::getLogFile() {
    return logFile;
}

const std::vector<std::string>& ConfigParser::getREPEndpoints() {
    return repEndpoints;
}

const std::vector<std::string>& ConfigParser::getPULLEndpoints() {
    return pullEndpoints;
}

const std::vector<std::string>& ConfigParser::getSUBEndpoints() {
    return subEndpoints;
}

const bool ConfigParser::isIPv4Only() {
    return ipv4Only;
}

const std::string& ConfigParser::getHTTPEndpoint() {
    return httpEndpoint;
}

uint64_t ConfigParser::getDefaultLRUCacheSize() {
    return defaultLRUCacheSize;
}

uint64_t ConfigParser::getDefaultTableBlockSize() {
    return defaultTableBlockSize;
}

uint64_t ConfigParser::getDefaultWriteBufferSize() {
    return defaultWriteBufferSize;
}

uint64_t ConfigParser::getDefaultBloomFilterBitsPerKey() {
    return defaultBloomFilterBitsPerKey;
}

bool ConfigParser::isCompressionEnabledPerDefault() {
    return compressionEnabledPerDefault;
}

int ConfigParser::getInternalHWM() {
    return internalHWM;
}

int ConfigParser::getExternalHWM() {
    return externalHWM;
}<|endoftext|>


<commit_before>#include <fstream>
#include <string>
#include <regex>
#include <unistd.h>
#include <vector>
#include <sstream>
#include <map>
#include <iostream>
#include <tclap/CmdLine.h>
#include "macros.hpp"
#include "ConfigParser.hpp"
#include "FileUtils.hpp"

namespace po = boost::program_options;
using std::string;
using std::vector;
using std::cout;
using std::cerr;
using std::endl;

/**
 * Split a comma-separated string into its components.
 * Does not handle quotes at all. Any comma is treated as a delimiter
 */
static void split(const std::string& source, std::vector<std::string>& elems, char delimiter=',') {
    std::stringstream ss(s);
    std::string item;
    while (std::getline(ss, item, delimiter)) {
        elems.push_back(item);
    }
    return elems;
}

/**
 * Check if a given ZMQ endpoint is a valid TCP or IPC endpoint
 */
static bool COLD checkTCPIPCEndpoint(const string& endpoint, bool allowIPC = true) {
    //Check if the parameter is valid
    bool isTCP = endpoint.find("tcp://") == 0;
    bool isIPC = endpoint.find("ipc://") == 0 && allowIPC;
    if(!isTCP && !isIPC) {
        cout << "Endpoint " << endpoint << " is not a valid TCP" << (allowIPC ? " or IPC": "") << " endpoint!" << endl;
        return false;
    }
    return true;
}

void COLD ConfigParser::saveConfigFile() {
    std::ofstream fout("yak.cfg");
    fout << "statistics-expunge-timeout=" << statisticsExpungeTimeout << '\n';
    fout << "static-file-path=" << staticFilePath << '\n';
    if(!logFile.empty()) {
        fout << "logfile=" << logFile << '\n';
    }
    for(string endpoint : repEndpoints) {
        fout << "rep-endpoint=" << endpoint << '\n';
    }
    for(string endpoint : pullEndpoints) {
        fout << "pull-endpoint=" << endpoint << '\n';
    }
    for(string endpoint : subEndpoints) {
        fout << "sub-endpoint=" << endpoint << '\n';
    }
    fout << "http-endpoint=" << httpEndpoint << '\n';
    if(ipv4Only) {
        fout << "ipv4-only" << '\n';
    }
    fout << "lru-cache-size=" << defaultLRUCacheSize << '\n';
    fout << "table-block-size=" << defaultTableBlockSize << '\n';
    fout << "write-buffer-size=" << defaultWriteBufferSize << '\n';
    fout << "bloom-filter-bits-per-key=" << defaultBloomFilterBitsPerKey << '\n';
    fout << "internal-hwm=" << internalHWM << '\n';
    fout << "external-hwm=" << externalHWM << '\n';
    if(!compressionEnabledPerDefault) {
        fout << "disable-compression=true" << '\n';
    }
    fout << "table-dir=" << tableSaveFolder << '\n';
    fout.close();
}

uint64_t ConfigParser::getStatisticsExpungeTimeout() {
    return statisticsExpungeTimeout;
}

/**
 * A buffer-overflow-safe readlink() wrapper for C++.
 * @return A string containing the readlink()ed filename, or
 *         an empty string with errno being set to the appropriate error.
 *         See the readlink() man(2) for errno details.
 */
static std::string safeReadlink(const std::string& filename) {
    size_t bufferSize = 255;

    //Increase buffer size until the buffer is large enough
    while (1) {
        char* buffer = new char[bufferSize];
        size_t rc = readlink (filename.c_str(), buffer, bufferSize);
        if (rc == -1) {
            delete[] buffer;
            if(errno == EINVAL) {
                //We know that bufsize is positive, so
                // the file is not a symlink.
                errno = 0;
                return filename;
            } else if(errno == ENAMETOOLONG) {
                bufferSize += 255;
            } else {
                //errno still contains the error value
                return "";
            }
        } else {
            //Success! rc == number of valid chars in buffer
            errno = 0;
            return string(buffer, rc);
        }
    }
}

COLD ConfigParser::ConfigParser(int argc, char** argv) {
    /**
     * Remember when adding options, saveConfigFile() must also be updated!
     */
    // Declare the supported options.
    string configFileName;
    repEndpoints = {"tcp://*:7100","ipc:///tmp/yakserver-rep"};
    pullEndpoints = {"tcp://*:7101","ipc:///tmp/yakserver-pull"};
    subEndpoints = {"tcp://*:7102","ipc:///tmp/yakserver-sub"};

    httpEndpoint = "tcp://*:7109";
    po::options_description generalOptions("General options");
    generalOptions.add_options()
        ("help,h", "Print help message")
        ("logfile,l", po::value<string>(&logFile)->default_value(""), "")
        ("config,c",
            po::value<string>(&configFileName)->default_value("yak.cfg"),
            "The configuration file to use")
        ("static-file-path",
            po::value<string>(&staticFilePath)->default_value("./static"),
            "The static file directory for HTML files")
        ("statistics-expunge-timeout",
            po::value<uint64_t>(&statisticsExpungeTimeout)->default_value(3600*1000)/*1 hour default*/,
            "The time in milliseconds statistics will be saved for retrieval after a job has finished.");
    po::options_description socketOptions("Socket options");
    socketOptions.add_options()
        ("req-endpoints,r",
            po::value<vector<string> >(&repEndpoints),
            "The endpoints the REP backend will bind to.\nDefaults to tcp://*:7100, ipc:///tmp/yakserver-rep")
        ("pull-endpoint,p", po::value<vector<string> >(&pullEndpoints),
            "The endpoints the PULL backend will bind to.\nDefaults to tcp://*:7101, ipc:///tmp/yakserver-pull")
        ("sub-endpoint,s", po::value<vector<string> >(&subEndpoints),
            "The endpoints the SUB backend will bind to.\nDefaults to tcp://*:7102, ipc:///tmp/yakserver-sub")
        ("http-endpoint,e", po::value<string>(&httpEndpoint),
            "The endpoint the internal HTTP server will listen on. Defaults to tcp://*:7109")
        ("ipv4-only,4","By default the application uses IPv6 sockets to bind to both IPv6 and IPv4. This option tells the application not to use IPv6 capable sockets.")
        ("external-hwm",
            po::value<int>(&externalHWM)->default_value(250),
            "Set the ZMQ High-watermark for external sockets")
        ("internal-hwm",
            po::value<int>(&internalHWM)->default_value(250),
            "Set the ZMQ High-watermark for internal sockets")
    ;
    po::options_description tableOptions("Table options");
    tableOptions.add_options()
        ("lru-cache-size",
            po::value<uint64_t>(&defaultLRUCacheSize)->default_value(1024 * 1024 * 16),
            "Set the default LRU cache size in bytes. Overriden by table-specific options.")
        ("table-block-size",
            po::value<uint64_t>(&defaultTableBlockSize)->default_value(256*1024),
            "Set the default table block size in bytes. Overriden by table-specific options.")
        ("write-buffer-size",
            po::value<uint64_t>(&defaultWriteBufferSize)->default_value(1024 * 1024 * 64),
            "Set the default write buffer size in bytes. Overriden by table-specific options.")
        ("bloom-filter-bits-per-key",
            po::value<uint64_t>(&defaultBloomFilterBitsPerKey)->default_value(0),
            "Set the default bits per key for the bloom filter. Set to 0 to disable bloom filter. Overriden by table-specific options.")
        ("disable-compression,d","By default table compression is enabled for all unconfigured tables. If this option is used, table compression is disabled by default. Overridden by table-specific options.")
        ("table-dir,t", po::value<string>(&tableSaveFolder)->default_value("./tables"), "The folder were the database tables should be saved to.")
    ;
    //
    try {
        TCLAP::CmdLine cmd("YakDBs", ' ', "1.0");
        TCLAP::ValueArg<std::string> logfileArg("l", "logfile", "The file to write the log to", false, "yakdb.log", "filename");
        TCLAP::ValueArg<std::string> configArg("c", "config", "The configuration file to read from", false, "", "filename");
        TCLAP::ValueArg<std::string> webuiArg("w", "webui", "The directory containing the static web user interface", false, "", "directory");
        //TCLAP::ValueArg<std::string> statisticsExpungeTimeoutArg("c", "config", "The configuration file to read from", false, "", "filename");
        TCLAP::MultiArg<std::string> reqEndpointsArg("r", "req-endpoints", "ZeroMQ endpoints for REQ requests (default: tcp://*:7100, ipc:///tmp/yakserver-rep)", false, "ZMQ endpoint");
        TCLAP::MultiArg<std::string> pullEndpointsArg("p", "pull-endpoints", "ZeroMQ endpoints for REQ requests (default: tcp://*:7101, ipc:///tmp/yakserver-pull)", false, "ZMQ endpoint");
        TCLAP::MultiArg<std::string> subEndpointsArg("s", "sub-endpoints", "ZeroMQ endpoints for REQ requests (default: tcp://*:7102, ipc:///tmp/yakserver-sub)", false, "ZMQ endpoint");
        TCLAP::ValueArg<std::string> logfileArg("h", "http-endpoint", "The HTTP server port to listen on", false, "", "port");
        TCLAP::ValueArg<std::string> logfileArg("4", "ipv4-only", "Use IPv4 sockets only (not IPv6)", false, "", "filename");
        TCLAP::ValueArg<std::string> logfileArg("c", "config", "The configuration file to read from", false, "", "filename");
    } catch (TCLAP::ArgException &e) {
        std::cerr << "Error: " << e.error() << " for arg " << e.argId() << std::endl;
    }
    //Create the main options group
    po::options_description desc("Options");
    desc.add(generalOptions).add(socketOptions).add(tableOptions);

    po::variables_map vm;
    po::store(po::parse_command_line(argc, argv, desc), vm);
    po::notify(vm);
    //Parse the config file
    bool processedConfigFile = false;
    if(fexists(configFileName)) {
        processedConfigFile = true;
        po::store(po::parse_config_file<char>(configFileName.c_str(), desc, true), vm);
    }
    po::notify(vm);
    //Check if --help is given
    if (vm.count("help")) {
        cout << desc << endl;
        exit(1);
    }
    //Check the endpoints
    for(string endpoint : repEndpoints) {
        if(!checkTCPIPCEndpoint(endpoint)) {
            cout << desc << endl;
            exit(1);
        }
    }
    for(string endpoint : pullEndpoints) {
        if(!checkTCPIPCEndpoint(endpoint)) {
            cout << desc << endl;
            exit(1);
        }
    }
    for(string endpoint : subEndpoints) {
        if(!checkTCPIPCEndpoint(endpoint)) {
            cout << desc << endl;
            exit(1);
        }
    }
    if(!checkTCPIPCEndpoint(httpEndpoint, false)) {
        cout << desc << endl;
        exit(1);
    }
    //Check directories & ensure symlinks are resolved properly
    // and they exist and are readable
    string originalStaticFilePath = staticFilePath;
    staticFilePath = safeReadlink(staticFilePath);
    if(unlikely(staticFilePath.size() == 0)) {
        if(errno == ENOENT) {
            cerr << "\x1B[33m[Warn] Static file directory '"
                 << originalStaticFilePath
                 << "' does not exist! The HTTP server definitely won't work.\x1B[0m" << endl;
        } else if (errno == EACCES) {
            cerr << "\x1B[33m[Warn] Static file directory '"
                 << originalStaticFilePath
                 << "' can't be used because the current user does not have the permission to read / list the directory! The HTTP server probably won't work.\x1B[0m" << endl;
        } else {
            cerr << "\x1B[33m[Warn] Unknown error '"
                 << strerror(errno)
                 << "' while trying to check the HTTP static file directory. The HTTP server probably won't work.\x1B[0m" << endl;
        }
    } else if(staticFilePath.back() != '/') {
        staticFilePath += "/";
    }
    //Get bool-ish options
    this->ipv4Only = (vm.count("ipv4-only") > 0);
    this->compressionEnabledPerDefault = (vm.count("disable-compression") > 0);
    //Write the config data to the config file unless there are no arguments
    if(argc > 1 || (processedConfigFile && configFileName != "yak.cfg")) {
        saveConfigFile();
    }
}

const std::string& ConfigParser::getLogFile() {
    return logFile;
}

const std::string& ConfigParser::getTableSaveFolderPath() {
    return tableSaveFolder;
}

const std::vector<std::string>& ConfigParser::getREPEndpoints() {
    return repEndpoints;
}

const std::vector<std::string>& ConfigParser::getPULLEndpoints() {
    return pullEndpoints;
}

const std::vector<std::string>& ConfigParser::getSUBEndpoints() {
    return subEndpoints;
}

const bool ConfigParser::isIPv4Only() {
    return ipv4Only;
}

const std::string& ConfigParser::getHTTPEndpoint() {
    return httpEndpoint;
}

uint64_t ConfigParser::getDefaultLRUCacheSize() {
    return defaultLRUCacheSize;
}

uint64_t ConfigParser::getDefaultTableBlockSize() {
    return defaultTableBlockSize;
}

uint64_t ConfigParser::getDefaultWriteBufferSize() {
    return defaultWriteBufferSize;
}

uint64_t ConfigParser::getDefaultBloomFilterBitsPerKey() {
    return defaultBloomFilterBitsPerKey;
}

bool ConfigParser::isCompressionEnabledPerDefault() {
    return compressionEnabledPerDefault;
}

int ConfigParser::getInternalHWM() {
    return internalHWM;
}

int ConfigParser::getExternalHWM() {
    return externalHWM;
}

std::string ConfigParser::getStaticFilePath() {
    return staticFilePath;
}<commit_msg>More CLI parser progress<commit_after>#include <fstream>
#include <string>
#include <regex>
#include <unistd.h>
#include <vector>
#include <sstream>
#include <map>
#include <iostream>
#include <tclap/CmdLine.h>
#include "macros.hpp"
#include "ConfigParser.hpp"
#include "FileUtils.hpp"

namespace po = boost::program_options;
using std::string;
using std::vector;
using std::cout;
using std::cerr;
using std::endl;

/**
 * Split a comma-separated string into its components.
 * Does not handle quotes at all. Any comma is treated as a delimiter
 */
static void split(const std::string& source, std::vector<std::string>& elems, char delimiter=',') {
    std::stringstream ss(s);
    std::string item;
    while (std::getline(ss, item, delimiter)) {
        elems.push_back(item);
    }
    return elems;
}

/**
 * Check if a given ZMQ endpoint is a valid TCP or IPC endpoint
 */
static bool COLD checkTCPIPCEndpoint(const string& endpoint, bool allowIPC = true) {
    //Check if the parameter is valid
    bool isTCP = endpoint.find("tcp://") == 0;
    bool isIPC = endpoint.find("ipc://") == 0 && allowIPC;
    if(!isTCP && !isIPC) {
        cout << "Endpoint " << endpoint << " is not a valid TCP" << (allowIPC ? " or IPC": "") << " endpoint!" << endl;
        return false;
    }
    return true;
}

void COLD ConfigParser::saveConfigFile() {
    std::ofstream fout("yak.cfg");
    fout << "statistics-expunge-timeout=" << statisticsExpungeTimeout << '\n';
    fout << "static-file-path=" << staticFilePath << '\n';
    if(!logFile.empty()) {
        fout << "logfile=" << logFile << '\n';
    }
    for(string endpoint : repEndpoints) {
        fout << "rep-endpoint=" << endpoint << '\n';
    }
    for(string endpoint : pullEndpoints) {
        fout << "pull-endpoint=" << endpoint << '\n';
    }
    for(string endpoint : subEndpoints) {
        fout << "sub-endpoint=" << endpoint << '\n';
    }
    fout << "http-endpoint=" << httpEndpoint << '\n';
    if(ipv4Only) {
        fout << "ipv4-only" << '\n';
    }
    fout << "lru-cache-size=" << defaultLRUCacheSize << '\n';
    fout << "table-block-size=" << defaultTableBlockSize << '\n';
    fout << "write-buffer-size=" << defaultWriteBufferSize << '\n';
    fout << "bloom-filter-bits-per-key=" << defaultBloomFilterBitsPerKey << '\n';
    fout << "internal-hwm=" << internalHWM << '\n';
    fout << "external-hwm=" << externalHWM << '\n';
    if(!compressionEnabledPerDefault) {
        fout << "disable-compression=true" << '\n';
    }
    fout << "table-dir=" << tableSaveFolder << '\n';
    fout.close();
}

uint64_t ConfigParser::getStatisticsExpungeTimeout() {
    return statisticsExpungeTimeout;
}

/**
 * A buffer-overflow-safe readlink() wrapper for C++.
 * @return A string containing the readlink()ed filename, or
 *         an empty string with errno being set to the appropriate error.
 *         See the readlink() man(2) for errno details.
 */
static std::string safeReadlink(const std::string& filename) {
    size_t bufferSize = 255;

    //Increase buffer size until the buffer is large enough
    while (1) {
        char* buffer = new char[bufferSize];
        size_t rc = readlink (filename.c_str(), buffer, bufferSize);
        if (rc == -1) {
            delete[] buffer;
            if(errno == EINVAL) {
                //We know that bufsize is positive, so
                // the file is not a symlink.
                errno = 0;
                return filename;
            } else if(errno == ENAMETOOLONG) {
                bufferSize += 255;
            } else {
                //errno still contains the error value
                return "";
            }
        } else {
            //Success! rc == number of valid chars in buffer
            errno = 0;
            return string(buffer, rc);
        }
    }
}

class ZMQEndpointConstraint : public TCLAP::Constraint<std::string> {
public:
    std::string description() {
        return "ZeroMQ endpoint constraint checker (TCP/IPC only)";
    }

    std::string shortID() {
        return "ZMQ Endpoint Constraint";
    }

    bool check(const std::string& value) {
        return checkTCPIPCEndpoint(value, true);
    }
}

COLD ConfigParser::ConfigParser(int argc, char** argv) {
    /**
     * Remember when adding options, saveConfigFile() must also be updated!
     */
    // Declare the supported options.
    string configFileName;
    repEndpoints = {"tcp://*:7100","ipc:///tmp/yakserver-rep"};
    pullEndpoints = {"tcp://*:7101","ipc:///tmp/yakserver-pull"};
    subEndpoints = {"tcp://*:7102","ipc:///tmp/yakserver-sub"};

    httpEndpoint = "tcp://*:7109";
    po::options_description generalOptions("General options");
    generalOptions.add_options()
        ("help,h", "Print help message")
        ("logfile,l", po::value<string>(&logFile)->default_value(""), "")
        ("config,c",
            po::value<string>(&configFileName)->default_value("yak.cfg"),
            "The configuration file to use")
        ("static-file-path",
            po::value<string>(&staticFilePath)->default_value("./static"),
            "The static file directory for HTML files")
        ("statistics-expunge-timeout",
            po::value<uint64_t>(&statisticsExpungeTimeout)->default_value(3600*1000)/*1 hour default*/,
            "The time in milliseconds statistics will be saved for retrieval after a job has finished.");
    po::options_description socketOptions("Socket options");
    socketOptions.add_options()
        ("req-endpoints,r",
            po::value<vector<string> >(&repEndpoints),
            "The endpoints the REP backend will bind to.\nDefaults to tcp://*:7100, ipc:///tmp/yakserver-rep")
        ("pull-endpoint,p", po::value<vector<string> >(&pullEndpoints),
            "The endpoints the PULL backend will bind to.\nDefaults to tcp://*:7101, ipc:///tmp/yakserver-pull")
        ("sub-endpoint,s", po::value<vector<string> >(&subEndpoints),
            "The endpoints the SUB backend will bind to.\nDefaults to tcp://*:7102, ipc:///tmp/yakserver-sub")
        ("http-endpoint,e", po::value<string>(&httpEndpoint),
            "The endpoint the internal HTTP server will listen on. Defaults to tcp://*:7109")
        ("ipv4-only,4","By default the application uses IPv6 sockets to bind to both IPv6 and IPv4. This option tells the application not to use IPv6 capable sockets.")
        ("external-hwm",
            po::value<int>(&externalHWM)->default_value(250),
            "Set the ZMQ High-watermark for external sockets")
        ("internal-hwm",
            po::value<int>(&internalHWM)->default_value(250),
            "Set the ZMQ High-watermark for internal sockets")
    ;
    po::options_description tableOptions("Table options");
    tableOptions.add_options()
        ("lru-cache-size",
            po::value<uint64_t>(&defaultLRUCacheSize)->default_value(1024 * 1024 * 16),
            "Set the default LRU cache size in bytes. Overriden by table-specific options.")
        ("table-block-size",
            po::value<uint64_t>(&defaultTableBlockSize)->default_value(256*1024),
            "Set the default table block size in bytes. Overriden by table-specific options.")
        ("write-buffer-size",
            po::value<uint64_t>(&defaultWriteBufferSize)->default_value(1024 * 1024 * 64),
            "Set the default write buffer size in bytes. Overriden by table-specific options.")
        ("bloom-filter-bits-per-key",
            po::value<uint64_t>(&defaultBloomFilterBitsPerKey)->default_value(0),
            "Set the default bits per key for the bloom filter. Set to 0 to disable bloom filter. Overriden by table-specific options.")
        ("disable-compression,d","By default table compression is enabled for all unconfigured tables. If this option is used, table compression is disabled by default. Overridden by table-specific options.")
        ("table-dir,t", po::value<string>(&tableSaveFolder)->default_value("./tables"), "The folder were the database tables should be saved to.")
    ;
    //
    try {
        TCLAP::CmdLine cmd("YakDB", ' ', "1.0");
        TCLAP::ValueArg<std::string> logfileArg("l", "logfile", "The file to write the log to", false, "yakdb.log", "filename");
        TCLAP::ValueArg<std::string> configArg("c", "config", "The configuration file to read from", false, "yakdb.cfg", "filename");
        TCLAP::ValueArg<std::string> webuiArg("w", "webui", "The directory containing the static web user interface", false, "./static", "directory");
        TCLAP::ValueArg<uint64_t> statisticsExpungeTimeoutArg("c", "config", "The configuration file to read from", false, 3600*1000, "milliseconds");
        ZMQEndpointConstraint endpointConstraint;
        TCLAP::MultiArg<std::string> reqEndpointsArg("r", "req-endpoints", "ZeroMQ endpoints for REQ requests (default: tcp://*:7100, ipc:///tmp/yakserver-rep)", false, endpointConstraint);
        TCLAP::MultiArg<std::string> pullEndpointsArg("p", "pull-endpoints", "ZeroMQ endpoints for REQ requests (default: tcp://*:7101, ipc:///tmp/yakserver-pull)", false, endpointConstraint);
        TCLAP::MultiArg<std::string> subEndpointsArg("s", "sub-endpoints", "ZeroMQ endpoints for REQ requests (default: tcp://*:7102, ipc:///tmp/yakserver-sub)", false, endpointConstraint);
        TCLAP::ValueArg<std::string> httpEndpointArg("h", "http-endpoint", "The HTTP server port to listen on", false, "", "port");
        TCLAP::SwitchArg ipv4OnlyArg("4", "ipv4-only", "Use IPv4 sockets only (disables IPv6)", cmd, false);

        TCLAP::ValueArg<std::string> externalHWMArg("e", "external-hwm", "External socket High Watermark", false, "", "messages");
        TCLAP::ValueArg<std::string> internalHWMArg("i", "internal-hwm", "Internal socket ", false, "", "filename");

        TCLAP::ValueArg<std::string> tableDirectoryArg("t", "table-directory", "The directory where the table data is stored", false, "", "directory");

        TCLAP::ValueArg<std::string> compressionModeArg("m", "compression-mode", "The default compression mode (none, bzip2, zlib or snappy)", false, "snappy", "compression mode");
        //Add arguments
        cmd.add(logfileArg);
        cmd.add(configArg);
        cmd.add(webuiArg);
        cmd.add(reqEndpointsArg);
        cmd.add(pullEndpointsArg);
        cmd.add(subEndpointsArg);
        cmd.add(httpEndpointArg);
        cmd.add(externalHWMArg);
        cmd.add(internalHWMArg);
        cmd.add(tableDirectoryArg);
        cmd.add(compressionModeArg);
        //Parse the commandline args
        cmd.parse(argc, argv);
        //Copy arguments to class instance
        this->logFile = logfileArg.getValue();
        this->configFileName = configArg.getValue();
        this->staticFilePath = webuiArg.getValue();
        statisticsExpungeTimeout = 3600*1000;

        repEndpoints
    } catch (TCLAP::ArgException &e) {
        std::cerr << "Error: " << e.error() << " for arg " << e.argId() << std::endl;
    }
    //Parse the config file
    bool processedConfigFile = false;
    if(fexists(configFileName)) {
        processedConfigFile = true;
        po::store(po::parse_config_file<char>(configFileName.c_str(), desc, true), vm);
    }
    po::notify(vm);
    //Check if --help is given
    if (vm.count("help")) {
        cout << desc << endl;
        exit(1);
    }
    //Check the endpoints
    for(string endpoint : repEndpoints) {
        if(!checkTCPIPCEndpoint(endpoint)) {
            cout << desc << endl;
            exit(1);
        }
    }
    for(string endpoint : pullEndpoints) {
        if(!checkTCPIPCEndpoint(endpoint)) {
            cout << desc << endl;
            exit(1);
        }
    }
    for(string endpoint : subEndpoints) {
        if(!checkTCPIPCEndpoint(endpoint)) {
            cout << desc << endl;
            exit(1);
        }
    }
    if(!checkTCPIPCEndpoint(httpEndpoint, false)) {
        cout << desc << endl;
        exit(1);
    }
    //Check directories & ensure symlinks are resolved properly
    // and they exist and are readable
    string originalStaticFilePath = staticFilePath;
    staticFilePath = safeReadlink(staticFilePath);
    if(unlikely(staticFilePath.size() == 0)) {
        if(errno == ENOENT) {
            cerr << "\x1B[33m[Warn] Static file directory '"
                 << originalStaticFilePath
                 << "' does not exist! The HTTP server definitely won't work.\x1B[0m" << endl;
        } else if (errno == EACCES) {
            cerr << "\x1B[33m[Warn] Static file directory '"
                 << originalStaticFilePath
                 << "' can't be used because the current user does not have the permission to read / list the directory! The HTTP server probably won't work.\x1B[0m" << endl;
        } else {
            cerr << "\x1B[33m[Warn] Unknown error '"
                 << strerror(errno)
                 << "' while trying to check the HTTP static file directory. The HTTP server probably won't work.\x1B[0m" << endl;
        }
    } else if(staticFilePath.back() != '/') {
        staticFilePath += "/";
    }
    //Get bool-ish options
    this->ipv4Only = (vm.count("ipv4-only") > 0);
    this->compressionEnabledPerDefault = (vm.count("disable-compression") > 0);
    //Write the config data to the config file unless there are no arguments
    if(argc > 1 || (processedConfigFile && configFileName != "yak.cfg")) {
        saveConfigFile();
    }
}

const std::string& ConfigParser::getLogFile() {
    return logFile;
}

const std::string& ConfigParser::getTableSaveFolderPath() {
    return tableSaveFolder;
}

const std::vector<std::string>& ConfigParser::getREPEndpoints() {
    return repEndpoints;
}

const std::vector<std::string>& ConfigParser::getPULLEndpoints() {
    return pullEndpoints;
}

const std::vector<std::string>& ConfigParser::getSUBEndpoints() {
    return subEndpoints;
}

const bool ConfigParser::isIPv4Only() {
    return ipv4Only;
}

const std::string& ConfigParser::getHTTPEndpoint() {
    return httpEndpoint;
}

uint64_t ConfigParser::getDefaultLRUCacheSize() {
    return defaultLRUCacheSize;
}

uint64_t ConfigParser::getDefaultTableBlockSize() {
    return defaultTableBlockSize;
}

uint64_t ConfigParser::getDefaultWriteBufferSize() {
    return defaultWriteBufferSize;
}

uint64_t ConfigParser::getDefaultBloomFilterBitsPerKey() {
    return defaultBloomFilterBitsPerKey;
}

bool ConfigParser::isCompressionEnabledPerDefault() {
    return compressionEnabledPerDefault;
}

int ConfigParser::getInternalHWM() {
    return internalHWM;
}

int ConfigParser::getExternalHWM() {
    return externalHWM;
}

std::string ConfigParser::getStaticFilePath() {
    return staticFilePath;
}<|endoftext|>


<commit_before>/* RTcmix  - Copyright (C) 2004  The RTcmix Development Team
   See ``AUTHORS'' for a list of contributors. See ``LICENSE'' for
   the license to this software and for a DISCLAIMER OF ALL WARRANTIES.
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <rtcmix_types.h>
#include <PField.h>
#include <RTcmixMIDI.h>
#include <RTMidiPField.h>
#include <ugens.h>		// for warn, die

//FIXME: better to have a struct with one field a string, the other type

#define NAME_VARIANTS 4
static char *_midi_type_name[][NAME_VARIANTS] = {
	// NB: Order of these must correspond to indices given by MIDIType enum!
	{ "cntl", "control", NULL, NULL },
	{ "noteonpitch", "onpitch", NULL, NULL },
	{ "noteonvel", "onvel", NULL, NULL },
	{ "noteoffpitch", "offpitch", NULL, NULL },
	{ "noteoffvel", "offvel", NULL, NULL },
	{ "bend", "pitchbend", NULL, NULL },
	{ "prog", "program", NULL, NULL },
	{ "chanpress", "monopress", "at", "aftertouch" },
	{ "polypress", "keypress", "polyat", "polyaftertouch" },
	{ NULL, NULL, NULL, NULL }
};

static char *_midi_controller_name[][NAME_VARIANTS] = {
	{ "banksel", NULL, NULL, NULL },
	{ "mod", "modwheel", "modulation", NULL },
	{ "breath", NULL, NULL, NULL },
// FIXME: This breaks _string_to_subtype!!
	{ NULL, NULL, NULL, NULL },
	{ "foot", NULL, NULL, NULL },
	{ "port time", "portamento time", NULL, NULL },
	{ "data", NULL, NULL, NULL },
	{ "vol", "volume", NULL, NULL },
	{ "bal", "balance", NULL, NULL },
	{ NULL, NULL, NULL, NULL },
	{ "pan", NULL, NULL, NULL },
	{ "exp", "expression", NULL, NULL },
	{ NULL, NULL, NULL, NULL }
//FIXME: add others
};

// -------------------------------------------------------- _midi_connection ---
//
//    midi = makeconnection("midi", min, max, default, lag, chan, type,
//                                                                [subtype])
//
//    <chan>      MIDI channel (1-16)
//
//    <type>      "noteonpitch", "noteonvel", "noteoffpitch", "noteoffpitch"
//                "cntl", "prog", "bend", "chanpress", "polypress"
//
//    <subtype>   depends on <type>:
//                   cntl       controller number or string symbol, such as
//                              "mod", "foot", "breath", "data", "volume", "pan"
//                   polypress  MIDI note number
//                   [no subtypes for the other types]
//

static RTNumberPField *
_midi_usage()
{
	die("makeconnection (midi)",
		"Usage: makeconnection(\"midi\", min, max, default, lag, "
		"chan, type[, subtype])");
	return NULL;
}

static MIDIType
_string_to_type(const char *type)
{
	for (int i = 0; _midi_type_name[i][0] != NULL; i++) {
		for (int j = 0; j < NAME_VARIANTS; j++) {
			const char *name = _midi_type_name[i][j];
			if (name == NULL)
				break;
			if (strcasecmp(type, name) == 0)
				return (MIDIType) i;
		}
	}
	return kMIDIInvalidType;
}

static MIDISubType
_string_to_subtype(const MIDIType type, const char *subtype)
{
	if (type == kMIDIControlType) {
		for (int i = 0; _midi_controller_name[i][0] != NULL; i++) {
			for (int j = 0; j < NAME_VARIANTS; j++) {
				const char *name = _midi_controller_name[i][j];
				if (name == NULL)
					break;
				if (strcasecmp(subtype, name) == 0)
					return (MIDISubType) i;
			}
		}
	}
	return kMIDIInvalidSubType;
}

RTNumberPField *
midi_connection(const Arg args[], const int nargs)
{
	static RTcmixMIDI *midiport = NULL;
	if (midiport == NULL)				// first time, so init midi system
		midiport = createMIDIPort();
	if (midiport == NULL)
		return NULL;

	double minval, maxval, defaultval, lag;
	int chan;
	MIDIType type = kMIDIInvalidType;
	MIDISubType subtype = kMIDIInvalidSubType;

	if (args[1].isType(DoubleType))
		minval = args[1];
	else
		return _midi_usage();

	if (args[2].isType(DoubleType))
		maxval = args[2];
	else
		return _midi_usage();

	if (args[3].isType(DoubleType))
		defaultval = args[3];
	else
		return _midi_usage();

	if (args[4].isType(DoubleType))
		lag = args[4];
	else
		return _midi_usage();
	if (lag < 0.0 || lag > 100.0) {
		die("makeconnection (midi)", "<lag> must be between 0 and 100");
		return NULL;
	}

	if (args[5].isType(DoubleType))
		chan = (int) args[5] - 1;		// convert to zero-based channel
	else
		return _midi_usage();
	if (chan < 0 || chan > 15) {
		die("makeconnection (midi)", "<chan> must be between 1 and 16");
		return NULL;
	}

	if (args[6].isType(StringType))
		type = _string_to_type(args[6]);
	else
		return _midi_usage();
	if (type == kMIDIInvalidType)
		return _midi_usage();

	if (nargs > 7) {
		if (args[7].isType(StringType))
			subtype = _string_to_subtype(type, args[7]);
		else if (args[7].isType(DoubleType))
			// NB: this can be a code or a literal int, e.g. note or controller num
			subtype = (MIDISubType) (int) args[7];
		else
			return _midi_usage();
		if (subtype == kMIDIInvalidSubType)
			return _midi_usage();
	}

	return new RTMidiPField(midiport, minval, maxval, defaultval, lag, chan,
																		type, subtype);
}

<commit_msg>Controller name improvements.<commit_after>/* RTcmix  - Copyright (C) 2004  The RTcmix Development Team
   See ``AUTHORS'' for a list of contributors. See ``LICENSE'' for
   the license to this software and for a DISCLAIMER OF ALL WARRANTIES.
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <rtcmix_types.h>
#include <Option.h>
#include <PField.h>
#include <RTcmixMIDI.h>
#include <RTMidiPField.h>
#include <ugens.h>		// for warn, die

//FIXME: better to have a struct with one field a string, the other type?

#define NAME_VARIANTS	4
static char *_midi_type_name[][NAME_VARIANTS] = {
	// NB: Order of these must correspond to indices given by MIDIType enum!
	{ "cntl", "control", NULL, NULL },
	{ "noteonpitch", "onpitch", NULL, NULL },
	{ "noteonvel", "onvel", NULL, NULL },
	{ "noteoffpitch", "offpitch", NULL, NULL },
	{ "noteoffvel", "offvel", NULL, NULL },
	{ "bend", "pitchbend", NULL, NULL },
	{ "prog", "program", NULL, NULL },
	{ "chanpress", "monopress", "at", "aftertouch" },
	{ "polypress", "keypress", "polyat", "polyaftertouch" },
	{ NULL, NULL, NULL, NULL }
};

static char *_midi_controller_name[128] = {
				"", "mod", "breath", "", "foot",
				"port time", "data", "volume", "balance", "",
/* 10 */		"pan", "expression", "fxctl1", "fxctl2", "",
				"", "gp1", "gp2", "gp3", "gp4",
/* 20 */		"", "", "", "", "",
				"", "", "", "", "",
/* 30 */		"", "", "", "", "",
				"", "", "", "", "",
/* 40 */		"", "", "", "", "",
				"", "", "", "", "",
/* 50 */		"", "", "", "", "",
				"", "", "", "", "",
/* 60 */		"", "", "", "", "sustainsw",
				"portamentosw", "sostenutosw", "softpedsw", "legatosw", "hold2sw",
/* 70 */		"sc1", "sc2", "sc3", "sc4", "sc5",
				"sc6", "sc7", "sc8", "sc9", "sc10",
/* 80 */		"gp5", "gp6", "gp7", "gp8", "portamento",
				"", "", "", "", "",
/* 90 */		"", "fx1depth", "fx2depth", "fx3depth", "fx4depth",
				"fx5depth", "dataincr", "datadecr", "nrplsb", "nrpmsb",
/* 100 */	"rplsb", "rpmsb", "", "", "",
				"", "", "", "", "",
/* 110 */	"", "", "", "", "",
				"", "", "", "", "",
/* 120 */	"sound off", "reset cntl", "local cntl", "notes off", "omni off",
				"omni on", "mono on", "poly on"
};

// --------------------------------------------------------- midi_connection ---
//
//    midi = makeconnection("midi", min, max, default, lag, chan, type,
//                                                                [subtype])
//
//    <chan>      MIDI channel (1-16)
//
//    <type>      "noteonpitch", "noteonvel", "noteoffpitch", "noteoffpitch"
//                "cntl", "prog", "bend", "chanpress", "polypress"
//
//    <subtype>   depends on <type>:
//                   cntl       controller number or string symbol, such as
//                              "mod", "foot", "breath", "data", "volume", "pan"
//                   polypress  MIDI note number
//                   [no subtypes for the other types]
//

static RTNumberPField *
_midi_usage()
{
	die("makeconnection (midi)",
		"Usage: makeconnection(\"midi\", min, max, default, lag, "
		"chan, type[, subtype])");
	return NULL;
}

static MIDIType
_string_to_type(const char *type)
{
	for (int i = 0; _midi_type_name[i][0] != NULL; i++) {
		for (int j = 0; j < NAME_VARIANTS; j++) {
			const char *name = _midi_type_name[i][j];
			if (name == NULL)
				break;
			if (strcasecmp(type, name) == 0)
				return (MIDIType) i;
		}
	}
	return kMIDIInvalidType;
}

static MIDISubType
_string_to_subtype(const MIDIType type, const char *subtype)
{
	if (type == kMIDIControlType) {
		for (int i = 0; i < 128; i++) {
			const char *name = _midi_controller_name[i];
			if (strcmp(subtype, name) == 0)
				return (MIDISubType) i;
		}
	}
	return kMIDIInvalidSubType;
}

/* Parse MIDI control name string, a sequence of colon-separated strings,
   with line-splicing recognition and some white-space eating.  Reads this:

      midi_control_names = " \
         bank select: \
         modulation wheel: \
         : \
         foot: \
         ...
         last name"

   resulting in these names: "bank select", "modulation wheel", "", "foot",
   ... "last name".  All except the empty "" override the builtin names.
*/
static void
_read_config()
{
#ifdef NOTYET
	char *names = strdup(Option::midiCntlNames());
	char *p, buf[128];
	int i = 0;
	int j = 0;
	bool eatspace = true;
	for (i = 0, p = names; i < 128 && *p != 0; p++) {
		if (eatspace && isspace(*p))
			continue;
		if (*p == '\\')
			continue;
		if (*p == ':') {
			buf[j] = 0;
			if (buf[0])
				_midi_controller_name[i] = strdup(buf);
			i++;
			j = 0;
			eatspace = true;
		}
		else {
			buf[j++] = *p;
			if (j == 127) {
				warn("makeconnection (midi)",
						"Controller name in config file too long");
				break;
			}
			eatspace = false;
		}
	}
	free(names);
#endif
}

RTNumberPField *
midi_connection(const Arg args[], const int nargs)
{
	static bool configRead = false;

	static RTcmixMIDI *midiport = NULL;
	if (midiport == NULL)				// first time, so init midi system
		midiport = createMIDIPort();
	if (midiport == NULL)
		return NULL;

	if (configRead == false) {
		_read_config();
		configRead = true;
	}

	double minval, maxval, defaultval, lag;
	int chan;
	MIDIType type = kMIDIInvalidType;
	MIDISubType subtype = kMIDIInvalidSubType;

	if (args[1].isType(DoubleType))
		minval = args[1];
	else
		return _midi_usage();

	if (args[2].isType(DoubleType))
		maxval = args[2];
	else
		return _midi_usage();

	if (args[3].isType(DoubleType))
		defaultval = args[3];
	else
		return _midi_usage();

	if (args[4].isType(DoubleType))
		lag = args[4];
	else
		return _midi_usage();
	if (lag < 0.0 || lag > 100.0) {
		die("makeconnection (midi)", "<lag> must be between 0 and 100");
		return NULL;
	}

	if (args[5].isType(DoubleType))
		chan = (int) args[5] - 1;		// convert to zero-based channel
	else
		return _midi_usage();
	if (chan < 0 || chan > 15) {
		die("makeconnection (midi)", "<chan> must be between 1 and 16");
		return NULL;
	}

	if (args[6].isType(StringType))
		type = _string_to_type(args[6]);
	else
		return _midi_usage();
	if (type == kMIDIInvalidType)
		return _midi_usage();

	if (nargs > 7) {
		if (args[7].isType(StringType))
			subtype = _string_to_subtype(type, args[7]);
		else if (args[7].isType(DoubleType))
			// NB: this can be a code or a literal int, e.g. note or controller num
			subtype = (MIDISubType) (int) args[7];
		else
			return _midi_usage();
		if (subtype == kMIDIInvalidSubType)
			return _midi_usage();
	}

	return new RTMidiPField(midiport, minval, maxval, defaultval, lag, chan,
																		type, subtype);
}

<|endoftext|>


<commit_before>#include <stdio.h>

#include <botan/cpuid.h>

using namespace Botan;

int main()
   {
   printf("Cache line size: %d\n", CPUID::cache_line_size());

   printf("RDTSC: %d\n", CPUID::has_rdtsc());
   printf("SSE2 %d\n", CPUID::has_sse2());
   printf("SSSE3 %d\n", CPUID::has_ssse3());
   printf("SSE41 %d\n", CPUID::has_sse41());
   printf("SSE42 %d\n", CPUID::has_sse42());
   printf("AES-NI %d\n", CPUID::has_aes_intel());

   printf("AES-VIA %d\n", CPUID::has_aes_via());

   printf("AltiVec %d\n", CPUID::has_altivec());
   }
<commit_msg>Clean up cpuid test prog<commit_after>#include <iostream>
#include <botan/cpuid.h>

using namespace Botan;

void print_if_feature(const std::string& feature_name, bool exists)
   {
   if(exists)
      std::cout << feature_name << '\n';
   else
      std::cout << '[' << feature_name << ']' << '\n';
   }

int main()
   {
   std::cout << "Cache line size = " << CPUID::cache_line_size() << "\n";

   print_if_feature("RDTSC", CPUID::has_rdtsc());
   print_if_feature("SSE2", CPUID::has_sse2());
   print_if_feature("SSSE3", CPUID::has_ssse3());
   print_if_feature("SSE4.1", CPUID::has_sse41());
   print_if_feature("SSE4.2", CPUID::has_sse42());

   print_if_feature("AES-NI", CPUID::has_aes_intel());
   print_if_feature("AES-VIA", CPUID::has_aes_via());

   print_if_feature("AltiVec", CPUID::has_altivec());
   }
<|endoftext|>


<commit_before>//=======================================================================
// Copyright (c) 2014-2017 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#pragma once

#include "dll/transform/transform_layer.hpp"

namespace dll {

/*!
 * \brief Batch Normalization layer
 */
template <typename Desc>
struct batch_normalization_2d_layer : transform_layer<batch_normalization_2d_layer<Desc>> {
    using desc      = Desc;                                             ///< The descriptor type
    using base_type = transform_layer<batch_normalization_2d_layer<Desc>>; ///< The base type

    static constexpr size_t Input = desc::Input; ///< The input size
    static constexpr float e      = 1e-8;        ///< Epsilon for numerical stability

    etl::fast_matrix<float, Input> gamma;
    etl::fast_matrix<float, Input> beta;

    etl::fast_matrix<float, Input> mean;
    etl::fast_matrix<float, Input> var;

    etl::fast_matrix<float, Input> last_mean;
    etl::fast_matrix<float, Input> last_var;

    etl::dyn_matrix<float, 2> input_pre; /// B x Input

    float momentum = 0.9;

    etl::fast_matrix<float, Input>& w = gamma;
    etl::fast_matrix<float, Input>& b = beta;

    batch_normalization_2d_layer() : base_type() {
        gamma = 1.0;
        beta = 1.0;
    }

    /*!
     * \brief Returns a string representation of the layer
     */
    static std::string to_short_string() {
        return "batch_norm";
    }

    using base_type::activate_hidden;

    /*!
     * \brief Apply the layer to the input
     * \param output The output
     * \param input The input to apply the layer to
     */
    template <typename Input, typename Output>
    static void activate_hidden(Output& output, const Input& input) {
        test_activate_hidden(output, input);
    }

    /*!
     * \brief Apply the layer to the input
     * \param output The output
     * \param input The input to apply the layer to
     */
    template <typename Input, typename Output>
    static void test_activate_hidden(Output& output, const Input& input) {
        output = input;
    }

    /*!
     * \brief Apply the layer to the input
     * \param output The output
     * \param input The input to apply the layer to
     */
    template <typename Input, typename Output>
    static void train_activate_hidden(Output& output, const Input& input) {
        output = input;

        // TODO
    }

    /*!
     * \brief Apply the layer to the batch of input
     * \return A batch of output corresponding to the activated input
     */
    template <typename V>
    auto batch_activate_hidden(const V& v) const {
        auto output = force_temporary_dim_only(v);
        test_batch_activate_hidden(output, v);
        return output;
    }

    /*!
     * \brief Apply the layer to the batch of input
     * \return A batch of output corresponding to the activated input
     */
    template <typename V>
    auto test_batch_activate_hidden(const V& v) const {
        auto output = force_temporary_dim_only(v);
        test_batch_activate_hidden(output, v);
        return output;
    }

    /*!
     * \brief Apply the layer to the batch of input
     * \param output The batch of output
     * \param input The batch of input to apply the layer to
     */
    template <typename Input, typename Output>
    void batch_activate_hidden(Output& output, const Input& input) const {
        test_batch_activate_hidden(output, input);
    }

    /*!
     * \brief Apply the layer to the batch of input
     * \param output The batch of output
     * \param input The batch of input to apply the layer to
     */
    template <typename Input, typename Output>
    void test_batch_activate_hidden(Output& output, const Input& input) const {
        const auto B = etl::dim<0>(input);

        auto mean_rep = etl::rep(mean, B);
        auto var_rep  = etl::rep(var, B);
        auto gamma_rep = etl::rep(gamma, B);
        auto beta_rep  = etl::rep(beta, B);

        output = (gamma_rep >> ((input - mean_rep) / etl::sqrt(var_rep + e))) + beta_rep;
    }

    /*!
     * \brief Apply the layer to the batch of input
     * \param output The batch of output
     * \param input The batch of input to apply the layer to
     */
    template <typename Input, typename Output>
    void train_batch_activate_hidden(Output& output, const Input& input) {
        const auto B = etl::dim<0>(input);

        last_mean     = etl::force_temporary(etl::mean_l(input));
        auto last_mean_rep = etl::rep(last_mean, B);

        last_var      = etl::force_temporary(etl::mean_l((input - last_mean_rep) >> (input - last_mean_rep)));
        auto last_var_rep  = etl::rep(last_var, B);

        input_pre = (input - last_mean_rep) / etl::sqrt(last_var_rep + e);

        auto gamma_rep = etl::rep(gamma, B);
        auto beta_rep  = etl::rep(beta, B);
        output         = (gamma_rep >> input_pre) + beta_rep;

        // Update the current mean and variance
        mean = momentum * mean + (1.0 - momentum) * last_mean;
        var  = momentum * var + (1.0 - momentum) * (B / (B - 1) * last_var);
    }

    /*!
     * \brief Adapt the errors, called before backpropagation of the errors.
     *
     * This must be used by layers that have both an activation fnction and a non-linearity.
     *
     * \param context the training context
     */
    template<typename C>
    void adapt_errors(C& context) const {
        cpp_unused(context);
    }

    /*!
     * \brief Backpropagate the errors to the previous layers
     * \param output The ETL expression into which write the output
     * \param context The training context
     */
    template<typename H, typename C>
    void backward_batch(H&& output, C& context) const {
        const auto B = etl::dim<0>(context.input);

        auto last_mean_rep = etl::rep(last_mean, B);
        auto last_var_rep  = etl::rep(last_var, B);
        auto gamma_rep     = etl::rep(gamma, B);

        auto& dy = context.errors;
        auto& h = context.input;

        output =
                 (1.0 / B) * gamma_rep >> etl::sqrt(last_var_rep + e)
            >>  ((B >> dy) - etl::rep(etl::sum_l(dy), B) - ((h - last_mean_rep) >> (1.0 / (last_var_rep + e)) >> etl::rep(etl::sum_l(dy >> (h - last_mean_rep)), B)));
    }

    /*!
     * \brief Compute the gradients for this layer, if any
     * \param context The trainng context
     */
    template<typename C>
    void compute_gradients(C& context) const {
        // Gradients of gamma
        context.w_grad = etl::sum_l(input_pre >> context.errors);

        // Gradients of beta
        context.b_grad = etl::sum_l(context.errors);

        cpp_unused(context);
    }
};

// Declare the traits for the layer

template<typename Desc>
struct layer_base_traits<batch_normalization_2d_layer<Desc>> {
    static constexpr bool is_neural     = false; ///< Indicates if the layer is a neural layer
    static constexpr bool is_dense      = false; ///< Indicates if the layer is dense
    static constexpr bool is_conv       = false; ///< Indicates if the layer is convolutional
    static constexpr bool is_deconv     = false; ///< Indicates if the layer is deconvolutional
    static constexpr bool is_standard   = false; ///< Indicates if the layer is standard
    static constexpr bool is_rbm        = false; ///< Indicates if the layer is RBM
    static constexpr bool is_pooling    = false; ///< Indicates if the layer is a pooling layer
    static constexpr bool is_unpooling  = false; ///< Indicates if the layer is an unpooling laye
    static constexpr bool is_transform  = true;  ///< Indicates if the layer is a transform layer
    static constexpr bool is_dynamic    = false; ///< Indicates if the layer is dynamic
    static constexpr bool pretrain_last = false; ///< Indicates if the layer is dynamic
    static constexpr bool sgd_supported = true;  ///< Indicates if the layer is supported by SGD
};

/*!
 * \brief Specialization of sgd_context for batch_normalization_2d_layer
 */
template <typename DBN, typename Desc, size_t L>
struct sgd_context<DBN, batch_normalization_2d_layer<Desc>, L> {
    using layer_t          = batch_normalization_2d_layer<Desc>;                            ///< The current layer type
    using previous_layer   = typename DBN::template layer_type<L - 1>;          ///< The previous layer type
    using previous_context = sgd_context<DBN, previous_layer, L - 1>;           ///< The previous layer's context
    using inputs_t         = decltype(std::declval<previous_context>().output); ///< The type of inputs

    inputs_t input;  ///< A batch of input
    inputs_t output; ///< A batch of output
    inputs_t errors; ///< A batch of errors

    etl::fast_matrix<float, Desc::Input> w_grad;
    etl::fast_matrix<float, Desc::Input> b_grad;

    sgd_context(layer_t& /*layer*/){}
};

} //end of dll namespace
<commit_msg>Cleanup<commit_after>//=======================================================================
// Copyright (c) 2014-2017 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#pragma once

#include "dll/transform/transform_layer.hpp"

namespace dll {

/*!
 * \brief Batch Normalization layer
 */
template <typename Desc>
struct batch_normalization_2d_layer : transform_layer<batch_normalization_2d_layer<Desc>> {
    using desc      = Desc;                                             ///< The descriptor type
    using base_type = transform_layer<batch_normalization_2d_layer<Desc>>; ///< The base type

    static constexpr size_t Input = desc::Input; ///< The input size
    static constexpr float e      = 1e-8;        ///< Epsilon for numerical stability

    etl::fast_matrix<float, Input> gamma;
    etl::fast_matrix<float, Input> beta;

    etl::fast_matrix<float, Input> mean;
    etl::fast_matrix<float, Input> var;

    etl::fast_matrix<float, Input> last_mean;
    etl::fast_matrix<float, Input> last_var;

    etl::dyn_matrix<float, 2> input_pre; /// B x Input

    float momentum = 0.9;

    etl::fast_matrix<float, Input>& w = gamma;
    etl::fast_matrix<float, Input>& b = beta;

    batch_normalization_2d_layer() : base_type() {
        gamma = 1.0;
        beta = 1.0;
    }

    /*!
     * \brief Returns a string representation of the layer
     */
    static std::string to_short_string() {
        return "batch_norm";
    }

    using base_type::activate_hidden;

    /*!
     * \brief Apply the layer to the input
     * \param output The output
     * \param input The input to apply the layer to
     */
    template <typename Input, typename Output>
    static void activate_hidden(Output& output, const Input& input) {
        test_activate_hidden(output, input);
    }

    /*!
     * \brief Apply the layer to the input
     * \param output The output
     * \param input The input to apply the layer to
     */
    template <typename Input, typename Output>
    static void test_activate_hidden(Output& output, const Input& input) {
        output = input;
    }

    /*!
     * \brief Apply the layer to the input
     * \param output The output
     * \param input The input to apply the layer to
     */
    template <typename Input, typename Output>
    static void train_activate_hidden(Output& output, const Input& input) {
        output = input;

        // TODO
    }

    /*!
     * \brief Apply the layer to the batch of input
     * \return A batch of output corresponding to the activated input
     */
    template <typename V>
    auto batch_activate_hidden(const V& v) const {
        auto output = force_temporary_dim_only(v);
        test_batch_activate_hidden(output, v);
        return output;
    }

    /*!
     * \brief Apply the layer to the batch of input
     * \return A batch of output corresponding to the activated input
     */
    template <typename V>
    auto test_batch_activate_hidden(const V& v) const {
        auto output = force_temporary_dim_only(v);
        test_batch_activate_hidden(output, v);
        return output;
    }

    /*!
     * \brief Apply the layer to the batch of input
     * \param output The batch of output
     * \param input The batch of input to apply the layer to
     */
    template <typename Input, typename Output>
    void batch_activate_hidden(Output& output, const Input& input) const {
        test_batch_activate_hidden(output, input);
    }

    /*!
     * \brief Apply the layer to the batch of input
     * \param output The batch of output
     * \param input The batch of input to apply the layer to
     */
    template <typename Input, typename Output>
    void test_batch_activate_hidden(Output& output, const Input& input) const {
        const auto B = etl::dim<0>(input);

        auto mean_rep = etl::rep(mean, B);
        auto var_rep  = etl::rep(var, B);
        auto gamma_rep = etl::rep(gamma, B);
        auto beta_rep  = etl::rep(beta, B);

        output = (gamma_rep >> ((input - mean_rep) / etl::sqrt(var_rep + e))) + beta_rep;
    }

    /*!
     * \brief Apply the layer to the batch of input
     * \param output The batch of output
     * \param input The batch of input to apply the layer to
     */
    template <typename Input, typename Output>
    void train_batch_activate_hidden(Output& output, const Input& input) {
        const auto B = etl::dim<0>(input);

        last_mean     = etl::mean_l(input);
        auto last_mean_rep = etl::rep(last_mean, B);

        last_var      = etl::mean_l((input - last_mean_rep) >> (input - last_mean_rep));
        auto last_var_rep  = etl::rep(last_var, B);

        input_pre = (input - last_mean_rep) / etl::sqrt(last_var_rep + e);

        auto gamma_rep = etl::rep(gamma, B);
        auto beta_rep  = etl::rep(beta, B);
        output         = (gamma_rep >> input_pre) + beta_rep;

        // Update the current mean and variance
        mean = momentum * mean + (1.0 - momentum) * last_mean;
        var  = momentum * var + (1.0 - momentum) * (B / (B - 1) * last_var);
    }

    /*!
     * \brief Adapt the errors, called before backpropagation of the errors.
     *
     * This must be used by layers that have both an activation fnction and a non-linearity.
     *
     * \param context the training context
     */
    template<typename C>
    void adapt_errors(C& context) const {
        cpp_unused(context);
    }

    /*!
     * \brief Backpropagate the errors to the previous layers
     * \param output The ETL expression into which write the output
     * \param context The training context
     */
    template<typename H, typename C>
    void backward_batch(H&& output, C& context) const {
        const auto B = etl::dim<0>(context.input);

        auto last_mean_rep = etl::rep(last_mean, B);
        auto last_var_rep  = etl::rep(last_var, B);
        auto gamma_rep     = etl::rep(gamma, B);

        auto& dy = context.errors;
        auto& h = context.input;

        output =
                 (1.0 / B) * gamma_rep >> etl::sqrt(last_var_rep + e)
            >>  ((B >> dy) - etl::rep(etl::sum_l(dy), B) - ((h - last_mean_rep) >> (1.0 / (last_var_rep + e)) >> etl::rep(etl::sum_l(dy >> (h - last_mean_rep)), B)));
    }

    /*!
     * \brief Compute the gradients for this layer, if any
     * \param context The trainng context
     */
    template<typename C>
    void compute_gradients(C& context) const {
        // Gradients of gamma
        context.w_grad = etl::sum_l(input_pre >> context.errors);

        // Gradients of beta
        context.b_grad = etl::sum_l(context.errors);
    }
};

// Declare the traits for the layer

template<typename Desc>
struct layer_base_traits<batch_normalization_2d_layer<Desc>> {
    static constexpr bool is_neural     = false; ///< Indicates if the layer is a neural layer
    static constexpr bool is_dense      = false; ///< Indicates if the layer is dense
    static constexpr bool is_conv       = false; ///< Indicates if the layer is convolutional
    static constexpr bool is_deconv     = false; ///< Indicates if the layer is deconvolutional
    static constexpr bool is_standard   = false; ///< Indicates if the layer is standard
    static constexpr bool is_rbm        = false; ///< Indicates if the layer is RBM
    static constexpr bool is_pooling    = false; ///< Indicates if the layer is a pooling layer
    static constexpr bool is_unpooling  = false; ///< Indicates if the layer is an unpooling laye
    static constexpr bool is_transform  = true;  ///< Indicates if the layer is a transform layer
    static constexpr bool is_dynamic    = false; ///< Indicates if the layer is dynamic
    static constexpr bool pretrain_last = false; ///< Indicates if the layer is dynamic
    static constexpr bool sgd_supported = true;  ///< Indicates if the layer is supported by SGD
};

/*!
 * \brief Specialization of sgd_context for batch_normalization_2d_layer
 */
template <typename DBN, typename Desc, size_t L>
struct sgd_context<DBN, batch_normalization_2d_layer<Desc>, L> {
    using layer_t          = batch_normalization_2d_layer<Desc>;                            ///< The current layer type
    using previous_layer   = typename DBN::template layer_type<L - 1>;          ///< The previous layer type
    using previous_context = sgd_context<DBN, previous_layer, L - 1>;           ///< The previous layer's context
    using inputs_t         = decltype(std::declval<previous_context>().output); ///< The type of inputs

    inputs_t input;  ///< A batch of input
    inputs_t output; ///< A batch of output
    inputs_t errors; ///< A batch of errors

    etl::fast_matrix<float, Desc::Input> w_grad;
    etl::fast_matrix<float, Desc::Input> b_grad;

    sgd_context(layer_t& /*layer*/){}
};

} //end of dll namespace
<|endoftext|>