<commit_before>#include <string>

#include <errno.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <netinet/tcp.h>

#include "client.h"

using namespace std;

namespace rpc {

void Future::wait() {
    Pthread_mutex_lock(&ready_m_);
    while (!ready_) {
        Pthread_cond_wait(&ready_cond_, &ready_m_);
    }
    Pthread_mutex_unlock(&ready_m_);
}

void Future::notify_ready() {
    Pthread_mutex_lock(&ready_m_);
    ready_ = true;
    Pthread_cond_signal(&ready_cond_);
    Pthread_mutex_unlock(&ready_m_);

    if (attr_.callback != NULL) {
        attr_.callback->run(this);

        // automatically cleanup the callback
        delete attr_.callback;
        attr_.callback = NULL;
    }
}

Client::Client(PollMgr* pollmgr): pollmgr_(pollmgr), sock_(-1), status_(NEW), bmark_(NULL) {
    Pthread_mutex_init(&pending_fu_m_, NULL);
    Pthread_mutex_init(&out_m_, NULL);
}

Client::~Client() {
    invalidate_pending_futures();

    Pthread_mutex_destroy(&pending_fu_m_);
    Pthread_mutex_destroy(&out_m_);

    //Log::debug("rpc::Client: destroyed");
}

void Client::invalidate_pending_futures() {
    list<Future*> futures;
    Pthread_mutex_lock(&pending_fu_m_);
    while (pending_fu_.empty() == false) {
        futures.push_back(pending_fu_.begin()->second);
        pending_fu_.erase(pending_fu_.begin());
    }
    Pthread_mutex_unlock(&pending_fu_m_);

    for (list<Future*>::iterator it = futures.begin(); it != futures.end(); ++it) {
        Future* fu = *it;
        if (fu != NULL) {
            fu->error_code_ = ENOTCONN;
            fu->notify_ready();

            // since we removed it from pending_fu_
            fu->release();
        }
    }

}

void Client::close() {
    if (status_ == CONNECTED) {
        pollmgr_->remove(this);
        ::close(sock_);
        status_ = CLOSED;

        invalidate_pending_futures();
    }
    status_ = CLOSED;
}

int Client::connect(const char* addr) {
    string addr_str(addr);
    size_t idx = addr_str.find(":");
    if (idx == string::npos) {
        Log::error("rpc::Client: bad connect address: %s", addr);
        errno = EINVAL;
        return -1;
    }
    string host = addr_str.substr(0, idx);
    string port = addr_str.substr(idx + 1);

    struct addrinfo hints, *result, *rp;
    memset(&hints, 0, sizeof(struct addrinfo));

    hints.ai_family = AF_INET; // ipv4
    hints.ai_socktype = SOCK_STREAM; // tcp

    int r = getaddrinfo(host.c_str(), port.c_str(), &hints, &result);
    if (r != 0) {
        Log::error("rpc::Client: getaddrinfo(): %s", gai_strerror(r));
        return -1;
    }

    for (rp = result; rp != NULL; rp = rp->ai_next) {
        sock_ = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
        if (sock_ == -1) {
            continue;
        }

        const int yes = 1;
        verify(setsockopt(sock_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) == 0);
        verify(setsockopt(sock_, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == 0);

        if (::connect(sock_, rp->ai_addr, rp->ai_addrlen) == 0) {
            break;
        }
        ::close(sock_);
        sock_ = -1;
    }
    freeaddrinfo(result);

    if (rp == NULL) {
        // failed to connect
        Log::error("rpc::Client: connect(): %s", strerror(errno));
        return -1;
    }

    verify(set_nonblocking(sock_, true) == 0);
    Log::info("rpc::Client: connected to %s", addr);

    status_ = CONNECTED;
    pollmgr_->add(this);

    return 0;
}

void Client::handle_error() {
    close();
}

void Client::handle_write() {
    if (status_ != CONNECTED) {
        return;
    }

    Pthread_mutex_lock(&out_m_);
    out_.write_to_fd(sock_);
    if (out_.empty()) {
        pollmgr_->update_mode(this, Pollable::READ);
    }
    Pthread_mutex_unlock(&out_m_);
}

void Client::handle_read() {
    if (status_ != CONNECTED) {
        return;
    }

    int bytes_read = in_.read_from_fd(sock_);
    if (bytes_read == 0) {
        return;
    }

    for (;;) {
        i32 packet_size;
        int n_peek = in_.peek(&packet_size, sizeof(i32));
        if (n_peek == sizeof(i32) && in_.content_size_gt(packet_size + sizeof(i32) - 1)) {
            // consume the packet size
            verify(in_.read(&packet_size, sizeof(i32)) == sizeof(i32));

            i64 reply_xid;
            i32 error_code;

            in_ >> reply_xid >> error_code;

            Pthread_mutex_lock(&pending_fu_m_);
            map<i64, Future*>::iterator it = pending_fu_.find(reply_xid);
            if (it != pending_fu_.end()) {
                Future* fu = it->second;
                verify(fu->xid_ == reply_xid);
                pending_fu_.erase(it);
                Pthread_mutex_unlock(&pending_fu_m_);

                fu->error_code_ = error_code;
                fu->reply_.read_from_marshal(in_, packet_size - sizeof(reply_xid) - sizeof(error_code));

                fu->notify_ready();

                // since we removed it from pending_fu_
                fu->release();
            }

        } else {
            // packet incomplete or no more packets to process
            break;
        }
    }
}

int Client::poll_mode() {
    int mode = Pollable::READ;
    Pthread_mutex_lock(&out_m_);
    if (!out_.empty()) {
        mode |= Pollable::WRITE;
    }
    Pthread_mutex_unlock(&out_m_);
    return mode;
}

Future* Client::begin_request(i32 rpc_id, const FutureAttr& attr /* =... */) {
    Pthread_mutex_lock(&out_m_);

    if (status_ != CONNECTED) {
        if (attr.callback != NULL) {
            delete attr.callback;
        }
        return NULL;
    }

    Future* fu = new Future(xid_counter_.next(), attr);
    Pthread_mutex_lock(&pending_fu_m_);
    pending_fu_[fu->xid_] = fu;
    pending_fu_.size();
    Pthread_mutex_unlock(&pending_fu_m_);

    // check if the client gets closed in the meantime
    if (status_ != CONNECTED) {
        Pthread_mutex_lock(&pending_fu_m_);
        map<i64, Future*>::iterator it = pending_fu_.find(fu->xid_);
        if (it != pending_fu_.end()) {
            it->second->release();
            pending_fu_.erase(it);
        }
        Pthread_mutex_unlock(&pending_fu_m_);

        return NULL;
    }

    bmark_ = out_.set_bookmark(sizeof(i32)); // will fill packet size later

    *this << fu->xid_;
    *this << rpc_id;

    // one ref is already in pending_fu_
    return (Future *) fu->ref_copy();
}

void Client::end_request() {
    // set reply size in packet
    if (bmark_ != NULL) {
        i32 request_size = out_.get_write_counter_and_reset();
        out_.write_bookmark(bmark_, &request_size);
        delete bmark_;
        bmark_ = NULL;
    }

    if (!out_.empty()) {
        pollmgr_->update_mode(this, Pollable::READ | Pollable::WRITE);
    }

    Pthread_mutex_unlock(&out_m_);
}

ClientPool::ClientPool(PollMgr* pollmgr /* =? */) {
    Pthread_mutex_init(&m_, NULL);

    if (pollmgr == NULL) {
        pollmgr_ = new PollMgr(1);
    } else {
        pollmgr_ = (PollMgr *) pollmgr->ref_copy();
    }
}

ClientPool::~ClientPool() {
    for (map<string, Client*>::iterator it = cache_.begin(); it != cache_.end(); ++it) {
        it->second->close_and_release();
    }

    pollmgr_->release();

    Pthread_mutex_destroy(&m_);
}

Client* ClientPool::get_client(const string& addr) {
    Client* cl = NULL;
    Pthread_mutex_lock(&m_);
    map<string, Client*>::iterator it = cache_.find(addr);
    if (it != cache_.end()) {
        cl = it->second;
    } else {
        cl = new Client(this->pollmgr_);
        if (cl->connect(addr.c_str()) != 0) {
            // connect failure
            cl->close_and_release();
            cl = NULL;
        } else {
            // connect success
            cache_[addr] = cl;
        }
    }
    Pthread_mutex_unlock(&m_);
    return cl;
}

}
<commit_msg>minor update<commit_after>#include <string>

#include <errno.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <netinet/tcp.h>

#include "client.h"

using namespace std;

namespace rpc {

void Future::wait() {
    Pthread_mutex_lock(&ready_m_);
    while (!ready_) {
        Pthread_cond_wait(&ready_cond_, &ready_m_);
    }
    Pthread_mutex_unlock(&ready_m_);
}

void Future::notify_ready() {
    Pthread_mutex_lock(&ready_m_);
    ready_ = true;
    Pthread_cond_signal(&ready_cond_);
    Pthread_mutex_unlock(&ready_m_);

    if (attr_.callback != NULL) {
        attr_.callback->run(this);

        // automatically cleanup the callback
        delete attr_.callback;
        attr_.callback = NULL;
    }
}

Client::Client(PollMgr* pollmgr): pollmgr_(pollmgr), sock_(-1), status_(NEW), bmark_(NULL) {
    Pthread_mutex_init(&pending_fu_m_, NULL);
    Pthread_mutex_init(&out_m_, NULL);
}

Client::~Client() {
    invalidate_pending_futures();

    Pthread_mutex_destroy(&pending_fu_m_);
    Pthread_mutex_destroy(&out_m_);

    //Log::debug("rpc::Client: destroyed");
}

void Client::invalidate_pending_futures() {
    list<Future*> futures;
    Pthread_mutex_lock(&pending_fu_m_);
    while (pending_fu_.empty() == false) {
        futures.push_back(pending_fu_.begin()->second);
        pending_fu_.erase(pending_fu_.begin());
    }
    Pthread_mutex_unlock(&pending_fu_m_);

    for (list<Future*>::iterator it = futures.begin(); it != futures.end(); ++it) {
        Future* fu = *it;
        if (fu != NULL) {
            fu->error_code_ = ENOTCONN;
            fu->notify_ready();

            // since we removed it from pending_fu_
            fu->release();
        }
    }

}

void Client::close() {
    if (status_ == CONNECTED) {
        pollmgr_->remove(this);
        ::close(sock_);
        status_ = CLOSED;

        invalidate_pending_futures();
    }
    status_ = CLOSED;
}

int Client::connect(const char* addr) {
    string addr_str(addr);
    size_t idx = addr_str.find(":");
    if (idx == string::npos) {
        Log::error("rpc::Client: bad connect address: %s", addr);
        errno = EINVAL;
        return -1;
    }
    string host = addr_str.substr(0, idx);
    string port = addr_str.substr(idx + 1);

    struct addrinfo hints, *result, *rp;
    memset(&hints, 0, sizeof(struct addrinfo));

    hints.ai_family = AF_INET; // ipv4
    hints.ai_socktype = SOCK_STREAM; // tcp

    int r = getaddrinfo(host.c_str(), port.c_str(), &hints, &result);
    if (r != 0) {
        Log::error("rpc::Client: getaddrinfo(): %s", gai_strerror(r));
        return -1;
    }

    for (rp = result; rp != NULL; rp = rp->ai_next) {
        sock_ = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
        if (sock_ == -1) {
            continue;
        }

        const int yes = 1;
        verify(setsockopt(sock_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) == 0);
        verify(setsockopt(sock_, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == 0);

        if (::connect(sock_, rp->ai_addr, rp->ai_addrlen) == 0) {
            break;
        }
        ::close(sock_);
        sock_ = -1;
    }
    freeaddrinfo(result);

    if (rp == NULL) {
        // failed to connect
        Log::error("rpc::Client: connect(): %s", strerror(errno));
        return -1;
    }

    verify(set_nonblocking(sock_, true) == 0);
    Log::info("rpc::Client: connected to %s", addr);

    status_ = CONNECTED;
    pollmgr_->add(this);

    return 0;
}

void Client::handle_error() {
    close();
}

void Client::handle_write() {
    if (status_ != CONNECTED) {
        return;
    }

    Pthread_mutex_lock(&out_m_);
    out_.write_to_fd(sock_);
    if (out_.empty()) {
        pollmgr_->update_mode(this, Pollable::READ);
    }
    Pthread_mutex_unlock(&out_m_);
}

void Client::handle_read() {
    if (status_ != CONNECTED) {
        return;
    }

    int bytes_read = in_.read_from_fd(sock_);
    if (bytes_read == 0) {
        return;
    }

    for (;;) {
        i32 packet_size;
        int n_peek = in_.peek(&packet_size, sizeof(i32));
        if (n_peek == sizeof(i32) && in_.content_size_gt(packet_size + sizeof(i32) - 1)) {
            // consume the packet size
            verify(in_.read(&packet_size, sizeof(i32)) == sizeof(i32));

            i64 reply_xid;
            i32 error_code;

            in_ >> reply_xid >> error_code;

            Pthread_mutex_lock(&pending_fu_m_);
            map<i64, Future*>::iterator it = pending_fu_.find(reply_xid);
            if (it != pending_fu_.end()) {
                Future* fu = it->second;
                verify(fu->xid_ == reply_xid);
                pending_fu_.erase(it);
                Pthread_mutex_unlock(&pending_fu_m_);

                fu->error_code_ = error_code;
                fu->reply_.read_from_marshal(in_, packet_size - sizeof(reply_xid) - sizeof(error_code));

                fu->notify_ready();

                // since we removed it from pending_fu_
                fu->release();
            }

        } else {
            // packet incomplete or no more packets to process
            break;
        }
    }
}

int Client::poll_mode() {
    int mode = Pollable::READ;
    Pthread_mutex_lock(&out_m_);
    if (!out_.empty()) {
        mode |= Pollable::WRITE;
    }
    Pthread_mutex_unlock(&out_m_);
    return mode;
}

Future* Client::begin_request(i32 rpc_id, const FutureAttr& attr /* =... */) {
    Pthread_mutex_lock(&out_m_);

    if (status_ != CONNECTED) {
        if (attr.callback != NULL) {
            delete attr.callback;
        }
        return NULL;
    }

    Future* fu = new Future(xid_counter_.next(), attr);
    Pthread_mutex_lock(&pending_fu_m_);
    pending_fu_[fu->xid_] = fu;
    pending_fu_.size();
    Pthread_mutex_unlock(&pending_fu_m_);

    // check if the client gets closed in the meantime
    if (status_ != CONNECTED) {
        Pthread_mutex_lock(&pending_fu_m_);
        map<i64, Future*>::iterator it = pending_fu_.find(fu->xid_);
        if (it != pending_fu_.end()) {
            it->second->release();
            pending_fu_.erase(it);
        }
        Pthread_mutex_unlock(&pending_fu_m_);

        return NULL;
    }

    bmark_ = out_.set_bookmark(sizeof(i32)); // will fill packet size later

    *this << fu->xid_;
    *this << rpc_id;

    // one ref is already in pending_fu_
    return (Future *) fu->ref_copy();
}

void Client::end_request() {
    // set reply size in packet
    if (bmark_ != NULL) {
        i32 request_size = out_.get_write_counter_and_reset();
        out_.write_bookmark(bmark_, &request_size);
        delete bmark_;
        bmark_ = NULL;
    }

    if (!out_.empty()) {
        pollmgr_->update_mode(this, Pollable::READ | Pollable::WRITE);
    }

    Pthread_mutex_unlock(&out_m_);
}

ClientPool::ClientPool(PollMgr* pollmgr /* =? */) {
    Pthread_mutex_init(&m_, NULL);

    if (pollmgr == NULL) {
        pollmgr_ = new PollMgr(1);
    } else {
        pollmgr_ = (PollMgr *) pollmgr->ref_copy();
    }
}

ClientPool::~ClientPool() {
    for (map<string, Client*>::iterator it = cache_.begin(); it != cache_.end(); ++it) {
        it->second->close_and_release();
    }
    pollmgr_->release();
    Pthread_mutex_destroy(&m_);
}

Client* ClientPool::get_client(const string& addr) {
    Client* cl = NULL;
    Pthread_mutex_lock(&m_);
    map<string, Client*>::iterator it = cache_.find(addr);
    if (it != cache_.end()) {
        cl = it->second;
    } else {
        cl = new Client(this->pollmgr_);
        if (cl->connect(addr.c_str()) != 0) {
            // connect failure
            cl->close_and_release();
            cl = NULL;
        } else {
            // connect success
            cache_.insert(make_pair(addr, cl));
        }
    }
    Pthread_mutex_unlock(&m_);
    return cl;
}

}
<|endoftext|>


<commit_before>#include <iostream>
#include <fstream>
#include <iterator>
#include <vector>

#include "IR.h"

/*
  save mem
  save reg
  mem node sets
  reg values
*/

void emit_int32_t(std::ofstream& f, int32_t data) {
  f.write((char*)&data,sizeof(data));
}

void emit_int_set(std::ofstream& f, int_set_t& s) {
  emit_int32_t(f, RES_INT_SET);
  emit_int32_t(f, s.size());
  for (auto& i: s) {
    emit_int32_t(f, i);
  }
}

void emit_node_item(std::ofstream& f, std::vector<int_set_t>& ni) {
  emit_int32_t(f, RES_NODE_ITEM);
  emit_int32_t(f, ni.size());
  for (auto& i: ni) {
    emit_int_set(f, i);
  }
}

void emit_node_set(std::ofstream& f, node_set_t& ns) {
  emit_int32_t(f, RES_NODE_SET);
  emit_int32_t(f, ns.size());
  for (auto& i: ns) {
    emit_int32_t(f, i.first);
    emit_node_item(f, i.second);
  }
}

void emit_value(std::ofstream& f, value_t& v) {
  emit_int32_t(f, RES_VALUE);
  emit_int_set(f, v.simple_type);
  emit_node_set(f, v.node_set);
}

void save_result(std::ofstream& f, int32_t iter_count, std::vector<node_set_t>& mem, std::vector<value_t>& reg) {
  emit_int32_t(f, iter_count);
  emit_int32_t(f, mem.size());
  emit_int32_t(f, reg.size());
  for (auto& i: mem) {
    emit_node_set(f, i);
  }
  for (auto& i: reg) {
    emit_value(f, i);
  }
}

void save_result_file(const char* name, int32_t iter_count, std::vector<node_set_t>& mem, std::vector<value_t>& reg) {
  //std::cout << "save result to: " << name << "\n";

  std::ofstream fout(name, std::ios::out | std::ios::binary);

  save_result(fout, iter_count, mem, reg);

  fout.close();
}
<commit_msg>save node set values in key order<commit_after>#include <iostream>
#include <fstream>
#include <iterator>
#include <vector>
#include <algorithm>

#include "IR.h"

/*
  save mem
  save reg
  mem node sets
  reg values
*/

void emit_int32_t(std::ofstream& f, int32_t data) {
  f.write((char*)&data,sizeof(data));
}

void emit_int_set(std::ofstream& f, int_set_t& s) {
  emit_int32_t(f, RES_INT_SET);
  emit_int32_t(f, s.size());
  for (auto& i: s) {
    emit_int32_t(f, i);
  }
}

void emit_node_item(std::ofstream& f, std::vector<int_set_t>& ni) {
  emit_int32_t(f, RES_NODE_ITEM);
  emit_int32_t(f, ni.size());
  for (auto& i: ni) {
    emit_int_set(f, i);
  }
}

void emit_node_set(std::ofstream& f, node_set_t& ns) {
  emit_int32_t(f, RES_NODE_SET);
  emit_int32_t(f, ns.size());

  // get sorted key vector
  std::vector<int32_t> keys;
  keys.reserve (ns.size());
  for (auto& it : ns) {
    keys.push_back(it.first);
  }
  std::sort (keys.begin(), keys.end());

  // save items in sorted key order
  for (auto& i: keys) {
    emit_int32_t(f, i);
    emit_node_item(f, ns[i]);
  }
}

void emit_value(std::ofstream& f, value_t& v) {
  emit_int32_t(f, RES_VALUE);
  emit_int_set(f, v.simple_type);
  emit_node_set(f, v.node_set);
}

void save_result(std::ofstream& f, int32_t iter_count, std::vector<node_set_t>& mem, std::vector<value_t>& reg) {
  emit_int32_t(f, iter_count);
  emit_int32_t(f, mem.size());
  emit_int32_t(f, reg.size());
  for (auto& i: mem) {
    emit_node_set(f, i);
  }
  for (auto& i: reg) {
    emit_value(f, i);
  }
}

void save_result_file(const char* name, int32_t iter_count, std::vector<node_set_t>& mem, std::vector<value_t>& reg) {
  //std::cout << "save result to: " << name << "\n";

  std::ofstream fout(name, std::ios::out | std::ios::binary);

  save_result(fout, iter_count, mem, reg);

  fout.close();
}
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2016 DeepCortex GmbH <legal@eventql.io>
 * Authors:
 *   - Paul Asmuth <paul@eventql.io>
 *
 * This program is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License ("the license") as
 * published by the Free Software Foundation, either version 3 of the License,
 * or any later version.
 *
 * In accordance with Section 7(e) of the license, the licensing of the Program
 * under the license does not imply a trademark license. Therefore any rights,
 * title and interest in our trademarks remain entirely with us.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the license for more details.
 *
 * You can be released from the requirements of the license by purchasing a
 * commercial license. Buying such a license is mandatory as soon as you develop
 * commercial activities involving this program without disclosing the source
 * code of your own applications
 */
#include <deque>
#include <thread>
#include <string.h>
#include <curl/curl.h>
#include <evcollect/evcollect.h>
#include <evcollect/util/logging.h>
#include <evcollect/util/base64.h>
#include <evcollect/util/time.h>
#include <evcollect/util/return_code.h>

namespace evcollect {
namespace plugin_eventql {

class EventQLTarget {
public:

  static const size_t kDefaultHTTPTimeoutMicros = 30 * kMicrosPerSecond;
  static const size_t kDefaultMaxQueueLength = 8192;

  EventQLTarget(
      const std::string& hostname,
      uint16_t port);

  ~EventQLTarget();

  void addRoute(
      const std::string& event_name_match,
      const std::string& target);

  void setHTTPTimeout(uint64_t usecs);
  void setMaxQueueLength(size_t queue_len);

  void setAuthToken(const std::string& auth_token);
  void setCredentials(
      const std::string& username,
      const std::string& password);

  ReturnCode emitEvent(const EventData& event);

  ReturnCode startUploadThread();
  void stopUploadThread();

protected:

  struct TargetTable {
    std::string database;
    std::string table;
  };

  struct EnqueuedEvent {
    std::string database;
    std::string table;
    std::string data;
  };

  struct EventRouting {
    std::string event_name_match;
    std::string target;
  };

  ReturnCode enqueueEvent(const EnqueuedEvent& event);
  bool awaitEvent(EnqueuedEvent* event);
  ReturnCode uploadEvent(const EnqueuedEvent& event);

  std::string hostname_;
  uint16_t port_;
  std::string username_;
  std::string password_;
  std::string auth_token_;
  std::deque<EnqueuedEvent> queue_;
  mutable std::mutex mutex_;
  mutable std::condition_variable cv_;
  size_t queue_max_length_;
  std::thread thread_;
  bool thread_running_;
  bool thread_shutdown_;
  std::vector<EventRouting> routes_;
  CURL* curl_;
  uint64_t http_timeout_;
};

EventQLTarget::EventQLTarget(
    const std::string& hostname,
    uint16_t port) :
    hostname_(hostname),
    port_(port),
    queue_max_length_(kDefaultMaxQueueLength),
    thread_running_(false),
    curl_(nullptr),
    http_timeout_(kDefaultHTTPTimeoutMicros) {
  curl_ = curl_easy_init();
}

EventQLTarget::~EventQLTarget() {
  if (curl_) {
    curl_easy_cleanup(curl_);
  }
}

void EventQLTarget::addRoute(
    const std::string& event_name_match,
    const std::string& target) {
  EventRouting r;
  r.event_name_match = event_name_match;
  r.target = target;
  routes_.emplace_back(r);
}

void EventQLTarget::setAuthToken(const std::string& auth_token) {
  auth_token_ = auth_token;
}

void EventQLTarget::setCredentials(
    const std::string& username,
    const std::string& password) {
  username_ = username;
  password_ = password;
}

void EventQLTarget::setHTTPTimeout(uint64_t usecs) {
  http_timeout_ = usecs;
}

void EventQLTarget::setMaxQueueLength(size_t queue_len) {
  queue_max_length_ = queue_len;
}

ReturnCode EventQLTarget::emitEvent(const EventData& event) {
  for (const auto& route : routes_) {
    if (route.event_name_match != event.event_name) {
      continue;
    }

    auto target = StringUtil::split(route.target, "/");
    if (target.size() != 2) {
      return ReturnCode::error(
          "EINVAL",
          "invalid target specification. " \
          "format is: database/table");
    }

    EnqueuedEvent e;
    e.database = target[0];
    e.table = target[1];
    e.data = event.event_data;

    auto rc = enqueueEvent(e);
    if (!rc.isSuccess()) {
      return rc;
    }
  }

  return ReturnCode::success();
}

ReturnCode EventQLTarget::enqueueEvent(const EnqueuedEvent& event) {
  std::unique_lock<std::mutex> lk(mutex_);

  while (queue_.size() >= queue_max_length_) {
    cv_.wait(lk);
  }

  queue_.emplace_back(event);
  cv_.notify_all();

  return ReturnCode::success();
}

bool EventQLTarget::awaitEvent(EnqueuedEvent* event) {
  std::unique_lock<std::mutex> lk(mutex_);

  if (queue_.size() == 0) {
    cv_.wait(lk);
  }

  if (queue_.size() == 0) {
    return false;
  } else {
    *event = queue_.front();
    queue_.pop_front();
    cv_.notify_all();
    return true;
  }
}

ReturnCode EventQLTarget::startUploadThread() {
  if (thread_running_) {
    return ReturnCode::error("RTERROR", "upload thread is already running");
  }

  auto upload_thread = [this] () {
    while (true) {
      {
        std::unique_lock<std::mutex> lk(mutex_);
        if (thread_shutdown_) {
          return;
        }
      }

      EnqueuedEvent ev;
      if (!awaitEvent(&ev)) {
        continue;
      }

      auto rc = uploadEvent(ev);
      if (!rc.isSuccess()) {
        logError(
            "error while uploading event to $0/$1: $2",
            ev.database,
            ev.table,
            rc.getMessage());
      }
    }
  };

  std::unique_lock<std::mutex> lk(mutex_);
  thread_running_ = true;
  thread_shutdown_ = false;
  thread_ = std::thread(upload_thread);
  return ReturnCode::success();
}

void EventQLTarget::stopUploadThread() {
  if (!thread_running_) {
    return;
  }

  thread_shutdown_ = true;
  cv_.notify_all();
  thread_.join();
  thread_running_ = false;
}

namespace {
size_t curl_write_cb(void* data, size_t size, size_t nmemb, std::string* s) {
  size_t pos = s->size();
  size_t len = pos + size * nmemb;
  s->resize(len);
  memcpy((char*) s->data() + pos, data, size * nmemb);
  return size * nmemb;
}
}

ReturnCode EventQLTarget::uploadEvent(const EnqueuedEvent& ev) {
  auto url = StringUtil::format(
      "http://$0:$1/api/v1/tables/insert",
      hostname_,
      port_);

  std::string body;
  body += "[{ ";
  body += StringUtil::format(
      "\"database\": \"$0\",",
      StringUtil::jsonEscape(ev.database));
  body += StringUtil::format(
      "\"table\": \"$0\"",
      StringUtil::jsonEscape(ev.table));
  body += "\"data\":" + ev.data;
  body += "}]";

  if (!curl_) {
    return ReturnCode::error("EIO", "curl_init() failed");
  }

  struct curl_slist* req_headers = NULL;
  req_headers = curl_slist_append(
      req_headers,
      "Content-Type: application/json; charset=utf-8");

  if (!auth_token_.empty()) {
    auto hdr = "Authorization: Token " + auth_token_;
    req_headers = curl_slist_append(req_headers, hdr.c_str());
  }

  if (!username_.empty() || !password_.empty()) {
    std::string hdr = "Authorization: Basic ";
    hdr += Base64::encode(username_ + ":" + password_);
    req_headers = curl_slist_append(req_headers, hdr.c_str());
  }

  std::string res_body;
  curl_easy_setopt(curl_, CURLOPT_URL, url.c_str());
  curl_easy_setopt(curl_, CURLOPT_TIMEOUT_MS, http_timeout_ / kMicrosPerMilli);
  curl_easy_setopt(curl_, CURLOPT_POSTFIELDS, body.c_str());
  curl_easy_setopt(curl_, CURLOPT_WRITEFUNCTION, curl_write_cb);
  curl_easy_setopt(curl_, CURLOPT_WRITEDATA, &res_body);
  CURLcode curl_res = curl_easy_perform(curl_);
  curl_slist_free_all(req_headers);
  if (curl_res != CURLE_OK) {
    return ReturnCode::error(
        "EIO",
        "http request failed: %s",
        curl_easy_strerror(curl_res));
  }

  long http_res_code = 0;
  curl_easy_getinfo(curl_, CURLINFO_RESPONSE_CODE, &http_res_code);

  switch (http_res_code) {
    case 201:
      return ReturnCode::success();
    case 400:
      return ReturnCode::error(
          "EINVAL",
          "http error: %li -- %.*s",
          http_res_code,
          res_body.size(),
          res_body.data());
    case 403:
    case 401:
      return ReturnCode::error(
          "EACCESS",
          "auth error: %li -- %.*s",
          http_res_code,
          res_body.size(),
          res_body.data());
    default:
      return ReturnCode::error(
          "EIO",
          "http error: %li -- %.*s",
          http_res_code,
          res_body.size(),
          res_body.data());
  }
}

bool pluginAttach(
    evcollect_ctx_t* ctx,
    evcollect_plugin_cfg_t* cfg,
    void** userdata) {

}

bool pluginEmitEvent(
    evcollect_ctx_t* ctx,
    void* userdata,
    const evcollect_event_t* event) {

}
//ReturnCode EventQLPlugin::pluginAttach(
//    const PropertyList& config,
//    void** userdata) {
//  std::string hostname = "localhost";
//  uint16_t port = 9175;
//
//  std::string hostname_opt;
//  if (config.get("port", &hostname_opt)) {
//    hostname = hostname_opt;
//  }
//
//  std::string port_opt;
//  if (config.get("port", &port_opt)) {
//    try {
//      port = std::stoul(port_opt);
//    } catch (...) {
//      return ReturnCode::error("EINVAL", "invalid port");
//    }
//  }
//
//  std::unique_ptr<EventQLTarget> target(new EventQLTarget(hostname, port));
//
//  std::string username;
//  std::string password;
//  config.get("username", &username);
//  config.get("password", &password);
//  if (!username.empty() || !password.empty()) {
//    target->setCredentials(username, password);
//  }
//
//  std::string auth_token;
//  if (config.get("auth_token", &auth_token)) {
//    target->setAuthToken(auth_token);
//  }
//
//  std::string http_timeout_opt;
//  if (config.get("http_timeout", &http_timeout_opt)) {
//    uint64_t http_timeout;
//    try {
//      http_timeout = std::stoull(http_timeout_opt);
//    } catch (...) {
//      return ReturnCode::error("EINVAL", "invalid value for http_timeout");
//    }
//    target->setHTTPTimeout(http_timeout);
//  }
//
//  std::string queue_maxlen_opt;
//  if (config.get("queue_maxlen", &queue_maxlen_opt)) {
//    uint64_t queue_maxlen;
//    try {
//      queue_maxlen = std::stoull(queue_maxlen_opt);
//    } catch (...) {
//      return ReturnCode::error("EINVAL", "invalid value for queue_maxlen");
//    }
//    target->setMaxQueueLength(queue_maxlen);
//  }
//
//  std::vector<std::vector<std::string>> route_cfg;
//  config.get("route", &route_cfg);
//  for (const auto& route : route_cfg) {
//    if (route.size() != 2) {
//      return ReturnCode::error(
//          "EINVAL",
//          "invalid number of arguments to route. " \
//          "format is: route <event> <target>");
//    }
//
//    target->addRoute(route[0], route[1]);
//  }
//
//  target->startUploadThread();
//  *userdata = target.release();
//  return ReturnCode::success();
//}
//
//void EventQLPlugin::pluginDetach(void* userdata) {
//  auto target = static_cast<EventQLTarget*>(userdata);
//  target->stopUploadThread();
//  delete target;
//}
//
//ReturnCode EventQLPlugin::pluginEmitEvent(
//    void* userdata,
//    const EventData& evdata) {
//  auto target = static_cast<EventQLTarget*>(userdata);
//  return target->emitEvent(evdata);
//}

} // namespace plugins_eventql
} // namespace evcollect

bool __evcollect_plugin_init(evcollect_ctx_t* ctx) {
  evcollect_output_plugin_register(
      ctx,
      "output",
      &evcollect::plugin_eventql::pluginEmitEvent);

  return true;
}

<commit_msg>fixes compile error<commit_after>/**
 * Copyright (c) 2016 DeepCortex GmbH <legal@eventql.io>
 * Authors:
 *   - Paul Asmuth <paul@eventql.io>
 *
 * This program is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License ("the license") as
 * published by the Free Software Foundation, either version 3 of the License,
 * or any later version.
 *
 * In accordance with Section 7(e) of the license, the licensing of the Program
 * under the license does not imply a trademark license. Therefore any rights,
 * title and interest in our trademarks remain entirely with us.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the license for more details.
 *
 * You can be released from the requirements of the license by purchasing a
 * commercial license. Buying such a license is mandatory as soon as you develop
 * commercial activities involving this program without disclosing the source
 * code of your own applications
 */
#include <deque>
#include <thread>
#include <condition_variable>
#include <string.h>
#include <curl/curl.h>
#include <evcollect/evcollect.h>
#include <evcollect/util/logging.h>
#include <evcollect/util/base64.h>
#include <evcollect/util/time.h>
#include <evcollect/util/return_code.h>

namespace evcollect {
namespace plugin_eventql {

class EventQLTarget {
public:

  static const size_t kDefaultHTTPTimeoutMicros = 30 * kMicrosPerSecond;
  static const size_t kDefaultMaxQueueLength = 8192;

  EventQLTarget(
      const std::string& hostname,
      uint16_t port);

  ~EventQLTarget();

  void addRoute(
      const std::string& event_name_match,
      const std::string& target);

  void setHTTPTimeout(uint64_t usecs);
  void setMaxQueueLength(size_t queue_len);

  void setAuthToken(const std::string& auth_token);
  void setCredentials(
      const std::string& username,
      const std::string& password);

  ReturnCode emitEvent(const EventData& event);

  ReturnCode startUploadThread();
  void stopUploadThread();

protected:

  struct TargetTable {
    std::string database;
    std::string table;
  };

  struct EnqueuedEvent {
    std::string database;
    std::string table;
    std::string data;
  };

  struct EventRouting {
    std::string event_name_match;
    std::string target;
  };

  ReturnCode enqueueEvent(const EnqueuedEvent& event);
  bool awaitEvent(EnqueuedEvent* event);
  ReturnCode uploadEvent(const EnqueuedEvent& event);

  std::string hostname_;
  uint16_t port_;
  std::string username_;
  std::string password_;
  std::string auth_token_;
  std::deque<EnqueuedEvent> queue_;
  mutable std::mutex mutex_;
  mutable std::condition_variable cv_;
  size_t queue_max_length_;
  std::thread thread_;
  bool thread_running_;
  bool thread_shutdown_;
  std::vector<EventRouting> routes_;
  CURL* curl_;
  uint64_t http_timeout_;
};

EventQLTarget::EventQLTarget(
    const std::string& hostname,
    uint16_t port) :
    hostname_(hostname),
    port_(port),
    queue_max_length_(kDefaultMaxQueueLength),
    thread_running_(false),
    curl_(nullptr),
    http_timeout_(kDefaultHTTPTimeoutMicros) {
  curl_ = curl_easy_init();
}

EventQLTarget::~EventQLTarget() {
  if (curl_) {
    curl_easy_cleanup(curl_);
  }
}

void EventQLTarget::addRoute(
    const std::string& event_name_match,
    const std::string& target) {
  EventRouting r;
  r.event_name_match = event_name_match;
  r.target = target;
  routes_.emplace_back(r);
}

void EventQLTarget::setAuthToken(const std::string& auth_token) {
  auth_token_ = auth_token;
}

void EventQLTarget::setCredentials(
    const std::string& username,
    const std::string& password) {
  username_ = username;
  password_ = password;
}

void EventQLTarget::setHTTPTimeout(uint64_t usecs) {
  http_timeout_ = usecs;
}

void EventQLTarget::setMaxQueueLength(size_t queue_len) {
  queue_max_length_ = queue_len;
}

ReturnCode EventQLTarget::emitEvent(const EventData& event) {
  for (const auto& route : routes_) {
    if (route.event_name_match != event.event_name) {
      continue;
    }

    auto target = StringUtil::split(route.target, "/");
    if (target.size() != 2) {
      return ReturnCode::error(
          "EINVAL",
          "invalid target specification. " \
          "format is: database/table");
    }

    EnqueuedEvent e;
    e.database = target[0];
    e.table = target[1];
    e.data = event.event_data;

    auto rc = enqueueEvent(e);
    if (!rc.isSuccess()) {
      return rc;
    }
  }

  return ReturnCode::success();
}

ReturnCode EventQLTarget::enqueueEvent(const EnqueuedEvent& event) {
  std::unique_lock<std::mutex> lk(mutex_);

  while (queue_.size() >= queue_max_length_) {
    cv_.wait(lk);
  }

  queue_.emplace_back(event);
  cv_.notify_all();

  return ReturnCode::success();
}

bool EventQLTarget::awaitEvent(EnqueuedEvent* event) {
  std::unique_lock<std::mutex> lk(mutex_);

  if (queue_.size() == 0) {
    cv_.wait(lk);
  }

  if (queue_.size() == 0) {
    return false;
  } else {
    *event = queue_.front();
    queue_.pop_front();
    cv_.notify_all();
    return true;
  }
}

ReturnCode EventQLTarget::startUploadThread() {
  if (thread_running_) {
    return ReturnCode::error("RTERROR", "upload thread is already running");
  }

  auto upload_thread = [this] () {
    while (true) {
      {
        std::unique_lock<std::mutex> lk(mutex_);
        if (thread_shutdown_) {
          return;
        }
      }

      EnqueuedEvent ev;
      if (!awaitEvent(&ev)) {
        continue;
      }

      auto rc = uploadEvent(ev);
      if (!rc.isSuccess()) {
        logError(
            "error while uploading event to $0/$1: $2",
            ev.database,
            ev.table,
            rc.getMessage());
      }
    }
  };

  std::unique_lock<std::mutex> lk(mutex_);
  thread_running_ = true;
  thread_shutdown_ = false;
  thread_ = std::thread(upload_thread);
  return ReturnCode::success();
}

void EventQLTarget::stopUploadThread() {
  if (!thread_running_) {
    return;
  }

  thread_shutdown_ = true;
  cv_.notify_all();
  thread_.join();
  thread_running_ = false;
}

namespace {
size_t curl_write_cb(void* data, size_t size, size_t nmemb, std::string* s) {
  size_t pos = s->size();
  size_t len = pos + size * nmemb;
  s->resize(len);
  memcpy((char*) s->data() + pos, data, size * nmemb);
  return size * nmemb;
}
}

ReturnCode EventQLTarget::uploadEvent(const EnqueuedEvent& ev) {
  auto url = StringUtil::format(
      "http://$0:$1/api/v1/tables/insert",
      hostname_,
      port_);

  std::string body;
  body += "[{ ";
  body += StringUtil::format(
      "\"database\": \"$0\",",
      StringUtil::jsonEscape(ev.database));
  body += StringUtil::format(
      "\"table\": \"$0\"",
      StringUtil::jsonEscape(ev.table));
  body += "\"data\":" + ev.data;
  body += "}]";

  if (!curl_) {
    return ReturnCode::error("EIO", "curl_init() failed");
  }

  struct curl_slist* req_headers = NULL;
  req_headers = curl_slist_append(
      req_headers,
      "Content-Type: application/json; charset=utf-8");

  if (!auth_token_.empty()) {
    auto hdr = "Authorization: Token " + auth_token_;
    req_headers = curl_slist_append(req_headers, hdr.c_str());
  }

  if (!username_.empty() || !password_.empty()) {
    std::string hdr = "Authorization: Basic ";
    hdr += Base64::encode(username_ + ":" + password_);
    req_headers = curl_slist_append(req_headers, hdr.c_str());
  }

  std::string res_body;
  curl_easy_setopt(curl_, CURLOPT_URL, url.c_str());
  curl_easy_setopt(curl_, CURLOPT_TIMEOUT_MS, http_timeout_ / kMicrosPerMilli);
  curl_easy_setopt(curl_, CURLOPT_POSTFIELDS, body.c_str());
  curl_easy_setopt(curl_, CURLOPT_WRITEFUNCTION, curl_write_cb);
  curl_easy_setopt(curl_, CURLOPT_WRITEDATA, &res_body);
  CURLcode curl_res = curl_easy_perform(curl_);
  curl_slist_free_all(req_headers);
  if (curl_res != CURLE_OK) {
    return ReturnCode::error(
        "EIO",
        "http request failed: %s",
        curl_easy_strerror(curl_res));
  }

  long http_res_code = 0;
  curl_easy_getinfo(curl_, CURLINFO_RESPONSE_CODE, &http_res_code);

  switch (http_res_code) {
    case 201:
      return ReturnCode::success();
    case 400:
      return ReturnCode::error(
          "EINVAL",
          "http error: %li -- %.*s",
          http_res_code,
          res_body.size(),
          res_body.data());
    case 403:
    case 401:
      return ReturnCode::error(
          "EACCESS",
          "auth error: %li -- %.*s",
          http_res_code,
          res_body.size(),
          res_body.data());
    default:
      return ReturnCode::error(
          "EIO",
          "http error: %li -- %.*s",
          http_res_code,
          res_body.size(),
          res_body.data());
  }
}

bool pluginAttach(
    evcollect_ctx_t* ctx,
    evcollect_plugin_cfg_t* cfg,
    void** userdata) {

}

bool pluginEmitEvent(
    evcollect_ctx_t* ctx,
    void* userdata,
    const evcollect_event_t* event) {

}
//ReturnCode EventQLPlugin::pluginAttach(
//    const PropertyList& config,
//    void** userdata) {
//  std::string hostname = "localhost";
//  uint16_t port = 9175;
//
//  std::string hostname_opt;
//  if (config.get("port", &hostname_opt)) {
//    hostname = hostname_opt;
//  }
//
//  std::string port_opt;
//  if (config.get("port", &port_opt)) {
//    try {
//      port = std::stoul(port_opt);
//    } catch (...) {
//      return ReturnCode::error("EINVAL", "invalid port");
//    }
//  }
//
//  std::unique_ptr<EventQLTarget> target(new EventQLTarget(hostname, port));
//
//  std::string username;
//  std::string password;
//  config.get("username", &username);
//  config.get("password", &password);
//  if (!username.empty() || !password.empty()) {
//    target->setCredentials(username, password);
//  }
//
//  std::string auth_token;
//  if (config.get("auth_token", &auth_token)) {
//    target->setAuthToken(auth_token);
//  }
//
//  std::string http_timeout_opt;
//  if (config.get("http_timeout", &http_timeout_opt)) {
//    uint64_t http_timeout;
//    try {
//      http_timeout = std::stoull(http_timeout_opt);
//    } catch (...) {
//      return ReturnCode::error("EINVAL", "invalid value for http_timeout");
//    }
//    target->setHTTPTimeout(http_timeout);
//  }
//
//  std::string queue_maxlen_opt;
//  if (config.get("queue_maxlen", &queue_maxlen_opt)) {
//    uint64_t queue_maxlen;
//    try {
//      queue_maxlen = std::stoull(queue_maxlen_opt);
//    } catch (...) {
//      return ReturnCode::error("EINVAL", "invalid value for queue_maxlen");
//    }
//    target->setMaxQueueLength(queue_maxlen);
//  }
//
//  std::vector<std::vector<std::string>> route_cfg;
//  config.get("route", &route_cfg);
//  for (const auto& route : route_cfg) {
//    if (route.size() != 2) {
//      return ReturnCode::error(
//          "EINVAL",
//          "invalid number of arguments to route. " \
//          "format is: route <event> <target>");
//    }
//
//    target->addRoute(route[0], route[1]);
//  }
//
//  target->startUploadThread();
//  *userdata = target.release();
//  return ReturnCode::success();
//}
//
//void EventQLPlugin::pluginDetach(void* userdata) {
//  auto target = static_cast<EventQLTarget*>(userdata);
//  target->stopUploadThread();
//  delete target;
//}
//
//ReturnCode EventQLPlugin::pluginEmitEvent(
//    void* userdata,
//    const EventData& evdata) {
//  auto target = static_cast<EventQLTarget*>(userdata);
//  return target->emitEvent(evdata);
//}

} // namespace plugins_eventql
} // namespace evcollect

bool __evcollect_plugin_init(evcollect_ctx_t* ctx) {
  evcollect_output_plugin_register(
      ctx,
      "output",
      &evcollect::plugin_eventql::pluginEmitEvent);

  return true;
}

<|endoftext|>


<commit_before>/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2006 Artem Pavlenko
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/

//$Id: postgis.cc 44 2005-04-22 18:53:54Z pavlenko $

#include <string>
#include <algorithm>
#include <set>
#include <sstream>
#include <iomanip>
#include <boost/lexical_cast.hpp>
#include "connection_manager.hpp"
#include "postgis.hpp"

DATASOURCE_PLUGIN(postgis_datasource)

const std::string postgis_datasource::GEOMETRY_COLUMNS="geometry_columns";
const std::string postgis_datasource::SPATIAL_REF_SYS="spatial_ref_system";

using std::clog;
using std::endl;

using boost::lexical_cast;
using boost::bad_lexical_cast;
using boost::shared_ptr;

postgis_datasource::postgis_datasource(parameters const& params)
    : datasource (params),
      table_(params.get("table")),
      type_(datasource::Vector),
      extent_initialized_(false),
      desc_(params.get("type")),
      creator_(params.get("host"),
               params.get("port"),
               params.get("dbname"),
               params.get("user"),
               params.get("password"))
{

    unsigned initial_size;
    unsigned max_size;

    try
    {
        initial_size = boost::lexical_cast<unsigned>(params_.get("initial_size"));
    }
    catch (bad_lexical_cast& )
    {
        initial_size = 1;
    }

    try
    {
        max_size = boost::lexical_cast<unsigned>(params_.get("max_size"));
    }
    catch (bad_lexical_cast&)
    {
        max_size = 10;
    }

    ConnectionManager *mgr=ConnectionManager::instance();
    mgr->registerPool(creator_, initial_size, max_size);

    shared_ptr<Pool<Connection,ConnectionCreator> > pool=mgr->getPool(creator_.id());
    if (pool)
    {
        shared_ptr<Connection> conn = pool->borrowObject();
        if (conn && conn->isOK())
        {
            PoolGuard<shared_ptr<Connection>,shared_ptr<Pool<Connection,ConnectionCreator> > > guard(conn,pool);
            std::string table_name=table_from_sql(table_);
            std::ostringstream s;
            s << "select f_geometry_column,srid,type from ";
            s << GEOMETRY_COLUMNS <<" where f_table_name='" << table_name<<"'";

            shared_ptr<ResultSet> rs=conn->executeQuery(s.str());

            if (rs->next())
            {
                try
                {
                    srid_ = lexical_cast<int>(rs->getValue("srid"));
                    desc_.set_srid(srid_);
                }
                catch (bad_lexical_cast &ex)
                {
                    clog << ex.what() << endl;
                }
                geometryColumn_=rs->getValue("f_geometry_column");
                std::string postgisType=rs->getValue("type");
            }
            rs->close();

            // collect attribute desc
            s.str("");
            s << "select * from "<<table_<<" limit 1";
            rs=conn->executeQuery(s.str());
            if (rs->next())
            {
                int count = rs->getNumFields();
                for (int i=0;i<count;++i)
                {
                    std::string fld_name=rs->getFieldName(i);
                    int length = rs->getFieldLength(i);

                    int type_oid = rs->getTypeOID(i);
                    switch (type_oid)
                    {
                    case 21:    // int2
                    case 23:    // int4
                        desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer,false,length));
                        break;
                    case 700:   // float4
                    case 701:   // float8
                        desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double,false,length));
                    case 1042:  // bpchar
                    case 1043:  // varchar
                    case 25:    // text
                        desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String));
                        break;
                    default: // shouldn't get here
#ifdef MAPNIK_DEBUG
                        clog << "unknown type_oid="<<type_oid<<endl;
#endif
                        break;
                    }
                }
            }
        }
    }
}

std::string const postgis_datasource::name_="postgis";

std::string postgis_datasource::name()
{
    return name_;
}

int postgis_datasource::type() const
{
    return type_;
}

layer_descriptor postgis_datasource::get_descriptor() const
{
    return desc_;
}

std::string postgis_datasource::table_from_sql(const std::string& sql)
{
    std::string table_name(sql);
    transform(table_name.begin(),table_name.end(),table_name.begin(),tolower);
    std::string::size_type idx=table_name.rfind("from");
    if (idx!=std::string::npos)
    {
        idx=table_name.find_first_not_of(" ",idx+4);
        table_name=table_name.substr(idx);
        idx=table_name.find_first_of(" )");
        return table_name.substr(0,idx);
    }
    return table_name;
}

featureset_ptr postgis_datasource::features(const query& q) const
{
    Envelope<double> const& box=q.get_bbox();
    ConnectionManager *mgr=ConnectionManager::instance();
    shared_ptr<Pool<Connection,ConnectionCreator> > pool=mgr->getPool(creator_.id());
    if (pool)
    {
        shared_ptr<Connection> conn = pool->borrowObject();
        if (conn && conn->isOK())
        {
            PoolGuard<shared_ptr<Connection>,shared_ptr<Pool<Connection,ConnectionCreator> > > guard(conn,pool);
            std::ostringstream s;

            s << "select asbinary("<<geometryColumn_<<") as geom";
            std::set<std::string> const& props=q.property_names();
            std::set<std::string>::const_iterator pos=props.begin();
            std::set<std::string>::const_iterator end=props.end();
            while (pos != end)
            {
                s <<",\""<<*pos<<"\"";
                ++pos;
            }
            s << " from " << table_<<" where "<<geometryColumn_<<" && setSRID('BOX3D(";
            s << std::setprecision(16);
            s << box.minx() << " " << box.miny() << ",";
            s << box.maxx() << " " << box.maxy() << ")'::box3d,"<<srid_<<")";

#ifdef MAPNIK_DEBUG
            std::clog << s.str() << "\n";
#endif
            shared_ptr<ResultSet> rs=conn->executeQuery(s.str(),1);
            return featureset_ptr(new postgis_featureset(rs,props.size()));
        }
    }
    return featureset_ptr();
}

featureset_ptr postgis_datasource::features_at_point(coord2d const& pt) const
{
    ConnectionManager *mgr=ConnectionManager::instance();
    shared_ptr<Pool<Connection,ConnectionCreator> > pool=mgr->getPool(creator_.id());
    if (pool)
    {
        shared_ptr<Connection> conn = pool->borrowObject();
        if (conn && conn->isOK())
        {
            PoolGuard<shared_ptr<Connection>,shared_ptr<Pool<Connection,ConnectionCreator> > > guard(conn,pool);
            std::ostringstream s;

            s << "select asbinary(" << geometryColumn_ << ") as geom";

            std::vector<attribute_descriptor>::const_iterator itr = desc_.get_descriptors().begin();
            std::vector<attribute_descriptor>::const_iterator end = desc_.get_descriptors().end();
            unsigned size=0;
            while (itr != end)
            {
                s <<",\""<< itr->get_name() << "\"";
                ++itr;
                ++size;
            }

            s << " from " << table_<<" where "<<geometryColumn_<<" && setSRID('BOX3D(";
            s << std::setprecision(16);
            s << pt.x << " " << pt.y << ",";
            s << pt.x << " " << pt.y << ")'::box3d,"<<srid_<<")";

#ifdef MAPNIK_DEBUG
            std::clog << s.str() << "\n";
#endif
            shared_ptr<ResultSet> rs=conn->executeQuery(s.str(),1);
            return featureset_ptr(new postgis_featureset(rs, size));
        }
    }
    return featureset_ptr();
}

Envelope<double> postgis_datasource::envelope() const
{
    if (extent_initialized_) return extent_;

    ConnectionManager *mgr=ConnectionManager::instance();
    shared_ptr<Pool<Connection,ConnectionCreator> > pool=mgr->getPool(creator_.id());
    if (pool)
    {
        shared_ptr<Connection> conn = pool->borrowObject();
        if (conn && conn->isOK())
        {
            std::ostringstream s;
            std::string table_name = table_from_sql(table_);
            if (params_.get("estimate_extent") == "true")
            {
                s << "select xmin(ext),ymin(ext),xmax(ext),ymax(ext)"
                  << " from (select estimated_extent('"
                  << table_name <<"','"
                  << geometryColumn_ << "') as ext) as tmp";
            }
            else
            {
                s << "select xmin(ext),ymin(ext),xmax(ext),ymax(ext)"
                  << " from (select extent(" <<geometryColumn_<< ") as ext from "
                  << table_name << ") as tmp";
            }

            shared_ptr<ResultSet> rs=conn->executeQuery(s.str());
            if (rs->next())
            {
                try
                {
                    double lox=lexical_cast<double>(rs->getValue(0));
                    double loy=lexical_cast<double>(rs->getValue(1));
                    double hix=lexical_cast<double>(rs->getValue(2));
                    double hiy=lexical_cast<double>(rs->getValue(3));
                    extent_.init(lox,loy,hix,hiy);
                    extent_initialized_ = true;
                }
                catch (bad_lexical_cast &ex)
                {
                    clog << ex.what() << endl;
                }
            }
            rs->close();
        }
    }
    return extent_;
}

postgis_datasource::~postgis_datasource() {}
<commit_msg>append namespace to 'transform' to support buiding with STLport.<commit_after>/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2006 Artem Pavlenko
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/

//$Id: postgis.cc 44 2005-04-22 18:53:54Z pavlenko $

#include <string>
#include <algorithm>
#include <set>
#include <sstream>
#include <iomanip>
#include <boost/lexical_cast.hpp>
#include "connection_manager.hpp"
#include "postgis.hpp"

DATASOURCE_PLUGIN(postgis_datasource)

const std::string postgis_datasource::GEOMETRY_COLUMNS="geometry_columns";
const std::string postgis_datasource::SPATIAL_REF_SYS="spatial_ref_system";

using std::clog;
using std::endl;

using boost::lexical_cast;
using boost::bad_lexical_cast;
using boost::shared_ptr;

postgis_datasource::postgis_datasource(parameters const& params)
    : datasource (params),
      table_(params.get("table")),
      type_(datasource::Vector),
      extent_initialized_(false),
      desc_(params.get("type")),
      creator_(params.get("host"),
               params.get("port"),
               params.get("dbname"),
               params.get("user"),
               params.get("password"))
{

    unsigned initial_size;
    unsigned max_size;

    try
    {
        initial_size = boost::lexical_cast<unsigned>(params_.get("initial_size"));
    }
    catch (bad_lexical_cast& )
    {
        initial_size = 1;
    }

    try
    {
        max_size = boost::lexical_cast<unsigned>(params_.get("max_size"));
    }
    catch (bad_lexical_cast&)
    {
        max_size = 10;
    }

    ConnectionManager *mgr=ConnectionManager::instance();
    mgr->registerPool(creator_, initial_size, max_size);

    shared_ptr<Pool<Connection,ConnectionCreator> > pool=mgr->getPool(creator_.id());
    if (pool)
    {
        shared_ptr<Connection> conn = pool->borrowObject();
        if (conn && conn->isOK())
        {
            PoolGuard<shared_ptr<Connection>,shared_ptr<Pool<Connection,ConnectionCreator> > > guard(conn,pool);
            std::string table_name=table_from_sql(table_);
            std::ostringstream s;
            s << "select f_geometry_column,srid,type from ";
            s << GEOMETRY_COLUMNS <<" where f_table_name='" << table_name<<"'";

            shared_ptr<ResultSet> rs=conn->executeQuery(s.str());

            if (rs->next())
            {
                try
                {
                    srid_ = lexical_cast<int>(rs->getValue("srid"));
                    desc_.set_srid(srid_);
                }
                catch (bad_lexical_cast &ex)
                {
                    clog << ex.what() << endl;
                }
                geometryColumn_=rs->getValue("f_geometry_column");
                std::string postgisType=rs->getValue("type");
            }
            rs->close();

            // collect attribute desc
            s.str("");
            s << "select * from "<<table_<<" limit 1";
            rs=conn->executeQuery(s.str());
            if (rs->next())
            {
                int count = rs->getNumFields();
                for (int i=0;i<count;++i)
                {
                    std::string fld_name=rs->getFieldName(i);
                    int length = rs->getFieldLength(i);

                    int type_oid = rs->getTypeOID(i);
                    switch (type_oid)
                    {
                    case 21:    // int2
                    case 23:    // int4
                        desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer,false,length));
                        break;
                    case 700:   // float4
                    case 701:   // float8
                        desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double,false,length));
                    case 1042:  // bpchar
                    case 1043:  // varchar
                    case 25:    // text
                        desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String));
                        break;
                    default: // shouldn't get here
#ifdef MAPNIK_DEBUG
                        clog << "unknown type_oid="<<type_oid<<endl;
#endif
                        break;
                    }
                }
            }
        }
    }
}

std::string const postgis_datasource::name_="postgis";

std::string postgis_datasource::name()
{
    return name_;
}

int postgis_datasource::type() const
{
    return type_;
}

layer_descriptor postgis_datasource::get_descriptor() const
{
    return desc_;
}

std::string postgis_datasource::table_from_sql(const std::string& sql)
{
    std::string table_name(sql);
    std::transform(table_name.begin(),table_name.end(),table_name.begin(),tolower);
    std::string::size_type idx=table_name.rfind("from");
    if (idx!=std::string::npos)
    {
        idx=table_name.find_first_not_of(" ",idx+4);
        table_name=table_name.substr(idx);
        idx=table_name.find_first_of(" )");
        return table_name.substr(0,idx);
    }
    return table_name;
}

featureset_ptr postgis_datasource::features(const query& q) const
{
    Envelope<double> const& box=q.get_bbox();
    ConnectionManager *mgr=ConnectionManager::instance();
    shared_ptr<Pool<Connection,ConnectionCreator> > pool=mgr->getPool(creator_.id());
    if (pool)
    {
        shared_ptr<Connection> conn = pool->borrowObject();
        if (conn && conn->isOK())
        {
            PoolGuard<shared_ptr<Connection>,shared_ptr<Pool<Connection,ConnectionCreator> > > guard(conn,pool);
            std::ostringstream s;

            s << "select asbinary("<<geometryColumn_<<") as geom";
            std::set<std::string> const& props=q.property_names();
            std::set<std::string>::const_iterator pos=props.begin();
            std::set<std::string>::const_iterator end=props.end();
            while (pos != end)
            {
                s <<",\""<<*pos<<"\"";
                ++pos;
            }
            s << " from " << table_<<" where "<<geometryColumn_<<" && setSRID('BOX3D(";
            s << std::setprecision(16);
            s << box.minx() << " " << box.miny() << ",";
            s << box.maxx() << " " << box.maxy() << ")'::box3d,"<<srid_<<")";

#ifdef MAPNIK_DEBUG
            std::clog << s.str() << "\n";
#endif
            shared_ptr<ResultSet> rs=conn->executeQuery(s.str(),1);
            return featureset_ptr(new postgis_featureset(rs,props.size()));
        }
    }
    return featureset_ptr();
}

featureset_ptr postgis_datasource::features_at_point(coord2d const& pt) const
{
    ConnectionManager *mgr=ConnectionManager::instance();
    shared_ptr<Pool<Connection,ConnectionCreator> > pool=mgr->getPool(creator_.id());
    if (pool)
    {
        shared_ptr<Connection> conn = pool->borrowObject();
        if (conn && conn->isOK())
        {
            PoolGuard<shared_ptr<Connection>,shared_ptr<Pool<Connection,ConnectionCreator> > > guard(conn,pool);
            std::ostringstream s;

            s << "select asbinary(" << geometryColumn_ << ") as geom";

            std::vector<attribute_descriptor>::const_iterator itr = desc_.get_descriptors().begin();
            std::vector<attribute_descriptor>::const_iterator end = desc_.get_descriptors().end();
            unsigned size=0;
            while (itr != end)
            {
                s <<",\""<< itr->get_name() << "\"";
                ++itr;
                ++size;
            }

            s << " from " << table_<<" where "<<geometryColumn_<<" && setSRID('BOX3D(";
            s << std::setprecision(16);
            s << pt.x << " " << pt.y << ",";
            s << pt.x << " " << pt.y << ")'::box3d,"<<srid_<<")";

#ifdef MAPNIK_DEBUG
            std::clog << s.str() << "\n";
#endif
            shared_ptr<ResultSet> rs=conn->executeQuery(s.str(),1);
            return featureset_ptr(new postgis_featureset(rs, size));
        }
    }
    return featureset_ptr();
}

Envelope<double> postgis_datasource::envelope() const
{
    if (extent_initialized_) return extent_;

    ConnectionManager *mgr=ConnectionManager::instance();
    shared_ptr<Pool<Connection,ConnectionCreator> > pool=mgr->getPool(creator_.id());
    if (pool)
    {
        shared_ptr<Connection> conn = pool->borrowObject();
        if (conn && conn->isOK())
        {
            std::ostringstream s;
            std::string table_name = table_from_sql(table_);
            if (params_.get("estimate_extent") == "true")
            {
                s << "select xmin(ext),ymin(ext),xmax(ext),ymax(ext)"
                  << " from (select estimated_extent('"
                  << table_name <<"','"
                  << geometryColumn_ << "') as ext) as tmp";
            }
            else
            {
                s << "select xmin(ext),ymin(ext),xmax(ext),ymax(ext)"
                  << " from (select extent(" <<geometryColumn_<< ") as ext from "
                  << table_name << ") as tmp";
            }

            shared_ptr<ResultSet> rs=conn->executeQuery(s.str());
            if (rs->next())
            {
                try
                {
                    double lox=lexical_cast<double>(rs->getValue(0));
                    double loy=lexical_cast<double>(rs->getValue(1));
                    double hix=lexical_cast<double>(rs->getValue(2));
                    double hiy=lexical_cast<double>(rs->getValue(3));
                    extent_.init(lox,loy,hix,hiy);
                    extent_initialized_ = true;
                }
                catch (bad_lexical_cast &ex)
                {
                    clog << ex.what() << endl;
                }
            }
            rs->close();
        }
    }
    return extent_;
}

postgis_datasource::~postgis_datasource() {}
<|endoftext|>


<commit_before>/**
 * A class which is responsible for storing words or phrases in a efficient
 * way. These vocabularies can be used for several NLP applications.
 *
 * @date                    1 April, 2015
 * @author                  Joeri HERMANS
 * @version                 0.1
 *
 * Copyright 2015 Joeri HERMANS
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// BEGIN Includes. ///////////////////////////////////////////////////

// System dependencies.
#include "../../ml/nlp/vocabulary.h"

#include <algorithm>
#include <cassert>

// Application dependencies.

// END Includes. /////////////////////////////////////////////////////

void Vocabulary::add( const std::string & entity ) {
    // Checking the preconditions.
    assert( !entity.empty() && !contains(entity) );

    mStorage.push_back(entity);
}

bool Vocabulary::contains( const std::string & entity ) const {
    return ( std::find(mStorage.begin(),mStorage.end(),entity)
                != mStorage.end() );
}

std::size_t Vocabulary::get( const std::string & entity ) const {
    std::vector<std::string>::const_iterator it;
    std::size_t index;

    it = std::find(mStorage.begin(),mStorage.end(),entity);
    index = std::distance(mStorage.begin(),it);

    return ( index );
}

std::size_t Vocabulary::size( void ) const {
    return ( mStorage.size() );
}
<commit_msg>Fix vocabulary includes.<commit_after>/**
 * A class which is responsible for storing words or phrases in a efficient
 * way. These vocabularies can be used for several NLP applications.
 *
 * @date                    1 April, 2015
 * @author                  Joeri HERMANS
 * @version                 0.1
 *
 * Copyright 2015 Joeri HERMANS
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// BEGIN Includes. ///////////////////////////////////////////////////

// System dependencies.
#include <algorithm>
#include <cassert>

// Application dependencies.
#include <ias/ml/nlp/vocabulary.h>

// END Includes. /////////////////////////////////////////////////////

void Vocabulary::add( const std::string & entity ) {
    // Checking the preconditions.
    assert( !entity.empty() && !contains(entity) );

    mStorage.push_back(entity);
}

bool Vocabulary::contains( const std::string & entity ) const {
    return ( std::find(mStorage.begin(),mStorage.end(),entity)
                != mStorage.end() );
}

std::size_t Vocabulary::get( const std::string & entity ) const {
    std::vector<std::string>::const_iterator it;
    std::size_t index;

    it = std::find(mStorage.begin(),mStorage.end(),entity);
    index = std::distance(mStorage.begin(),it);

    return ( index );
}

std::size_t Vocabulary::size( void ) const {
    return ( mStorage.size() );
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2018 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <base58.h>

#include <hash.h>
#include <uint256.h>

#include <assert.h>
#include <stdint.h>
#include <string.h>
#include <vector>
#include <string>
#include <boost/variant/apply_visitor.hpp>
#include <boost/variant/static_visitor.hpp>

/** All alphanumeric characters except for "0", "I", "O", and "l" */
/** All alphanumeric characters except for "0", "I", "O", and "l" */
static const char* pszBase58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";

bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch)
{
    // Skip leading spaces.
    while (*psz && isspace(*psz))
        psz++;
    // Skip and count leading '1's.
    int zeroes = 0;
    int length = 0;
    while (*psz == '1') {
        zeroes++;
        psz++;
    }
    // Allocate enough space in big-endian base256 representation.
    int size = strlen(psz) * 733 /1000 + 1; // log(58) / log(256), rounded up.
    std::vector<unsigned char> b256(size);
    // Process the characters.
    while (*psz && !isspace(*psz)) {
        // Decode base58 character
        const char* ch = strchr(pszBase58, *psz);
        if (ch == nullptr)
            return false;
        // Apply "b256 = b256 * 58 + ch".
        int carry = ch - pszBase58;
        int i = 0;
        for (std::vector<unsigned char>::reverse_iterator it = b256.rbegin(); (carry != 0 || i < length) && (it != b256.rend()); ++it, ++i) {
            carry += 58 * (*it);
            *it = carry % 256;
            carry /= 256;
        }
        assert(carry == 0);
        length = i;
        psz++;
    }
    // Skip trailing spaces.
    while (isspace(*psz))
        psz++;
    if (*psz != 0)
        return false;
    // Skip leading zeroes in b256.
    std::vector<unsigned char>::iterator it = b256.begin() + (size - length);
    while (it != b256.end() && *it == 0)
        it++;
    // Copy result into output vector.
    vch.reserve(zeroes + (b256.end() - it));
    vch.assign(zeroes, 0x00);
    while (it != b256.end())
        vch.push_back(*(it++));
    return true;
}

std::string EncodeBase58(const unsigned char* pbegin, const unsigned char* pend)
{
    // Skip & count leading zeroes.
    int zeroes = 0;
    int length = 0;
    while (pbegin != pend && *pbegin == 0) {
        pbegin++;
        zeroes++;
    }
    // Allocate enough space in big-endian base58 representation.
    int size = (pend - pbegin) * 138 / 100 + 1; // log(256) / log(58), rounded up.
    std::vector<unsigned char> b58(size);
    // Process the bytes.
    while (pbegin != pend) {
        int carry = *pbegin;
        int i = 0;
        // Apply "b58 = b58 * 256 + ch".
        for (std::vector<unsigned char>::reverse_iterator it = b58.rbegin(); (carry != 0 || i < length) && (it != b58.rend()); it++, i++) {
            carry += 256 * (*it);
            *it = carry % 58;
            carry /= 58;
        }

        assert(carry == 0);
        length = i;
        pbegin++;
    }
    // Skip leading zeroes in base58 result.
    std::vector<unsigned char>::iterator it = b58.begin() + (size - length);
    while (it != b58.end() && *it == 0)
        it++;
    // Translate the result into a string.
    std::string str;
    str.reserve(zeroes + (b58.end() - it));
    str.assign(zeroes, '1');
    while (it != b58.end())
        str += pszBase58[*(it++)];
    return str;
}

std::string EncodeBase58(const std::vector<unsigned char>& vch)
{
    return EncodeBase58(vch.data(), vch.data() + vch.size());
}

bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet)
{
    return DecodeBase58(str.c_str(), vchRet);
}

std::string EncodeBase58Check(const std::vector<unsigned char>& vchIn)
{
    // add 4-byte hash check to the end
    std::vector<unsigned char> vch(vchIn);
    uint256 hash = Hash(vch.begin(), vch.end());
    vch.insert(vch.end(), (unsigned char*)&hash, (unsigned char*)&hash + 4);
    return EncodeBase58(vch);
}

bool DecodeBase58Check(const char* psz, std::vector<unsigned char>& vchRet)
{
    if (!DecodeBase58(psz, vchRet) ||
        (vchRet.size() < 4)) {
        vchRet.clear();
        return false;
    }
    // re-calculate the checksum, ensure it matches the included 4-byte checksum
    uint256 hash = Hash(vchRet.begin(), vchRet.end() - 4);
    if (memcmp(&hash, &vchRet.end()[-4], 4) != 0) {
        vchRet.clear();
        return false;
    }
    vchRet.resize(vchRet.size() - 4);
    return true;
}

bool DecodeBase58Check(const std::string& str, std::vector<unsigned char>& vchRet)
{
    return DecodeBase58Check(str.c_str(), vchRet);
}

CBase58Data::CBase58Data()
{
    vchVersion.clear();
    vchData.clear();
}

void CBase58Data::SetData(const std::vector<unsigned char>& vchVersionIn, const void* pdata, size_t nSize)
{
    vchVersion = vchVersionIn;
    vchData.resize(nSize);
    if (!vchData.empty())
        memcpy(vchData.data(), pdata, nSize);
}

void CBase58Data::SetData(const std::vector<unsigned char>& vchVersionIn, const unsigned char* pbegin, const unsigned char* pend)
{
    SetData(vchVersionIn, (void*)pbegin, pend - pbegin);
}

bool CBase58Data::SetString(const char* psz, unsigned int nVersionBytes)
{
    std::vector<unsigned char> vchTemp;
    bool rc58 = DecodeBase58Check(psz, vchTemp);
    if ((!rc58) || (vchTemp.size() < nVersionBytes)) {
        vchData.clear();
        vchVersion.clear();
        return false;
    }
    vchVersion.assign(vchTemp.begin(), vchTemp.begin() + nVersionBytes);
    vchData.resize(vchTemp.size() - nVersionBytes);
    if (!vchData.empty())
        memcpy(vchData.data(), vchTemp.data() + nVersionBytes, vchData.size());
    memory_cleanse(vchTemp.data(), vchTemp.size());
    return true;
}

bool CBase58Data::SetString(const std::string& str)
{
    return SetString(str.c_str());
}

std::string CBase58Data::ToString() const
{
    std::vector<unsigned char> vch = vchVersion;
    vch.insert(vch.end(), vchData.begin(), vchData.end());
    return EncodeBase58Check(vch);
}

int CBase58Data::CompareTo(const CBase58Data& b58) const
{
    if (vchVersion < b58.vchVersion)
        return -1;
    if (vchVersion > b58.vchVersion)
        return 1;
    if (vchData < b58.vchData)
        return -1;
    if (vchData > b58.vchData)
        return 1;
    return 0;
}

namespace
{
/** base58-encoded Chaincoin addresses.
 * Public-key-hash-addresses have version 0 (or 111 testnet).
 * The data vector contains RIPEMD160(SHA256(pubkey)), where pubkey is the serialized public key.
 * Script-hash-addresses have version 5 (or 196 testnet).
 * The data vector contains RIPEMD160(SHA256(cscript)), where cscript is the serialized redemption script.
 */
class CBitcoinAddress : public CBase58Data {
public:
    bool Set(const CKeyID &id);
    bool Set(const CScriptID &id);
    bool Set(const CTxDestination &dest);
    bool IsValid() const;
    bool IsValid(const CChainParams &params) const;

    CBitcoinAddress() {}
    CBitcoinAddress(const CTxDestination &dest) { Set(dest); }
    CBitcoinAddress(const std::string& strAddress) { SetString(strAddress); }
    CBitcoinAddress(const char* pszAddress) { SetString(pszAddress); }

    CTxDestination Get() const;
    bool GetKeyID(CKeyID &keyID) const;
    bool IsScript() const;
};

class CBitcoinAddressVisitor : public boost::static_visitor<bool>
{
private:
    CBitcoinAddress* addr;

public:
    explicit CBitcoinAddressVisitor(CBitcoinAddress* addrIn) : addr(addrIn) {}

    bool operator()(const CKeyID& id) const { return addr->Set(id); }
    bool operator()(const CScriptID& id) const { return addr->Set(id); }
    bool operator()(const CNoDestination& no) const { return false; }
};

} // namespace

bool CBitcoinAddress::Set(const CKeyID& id)
{
    SetData(Params().Base58Prefix(CChainParams::PUBKEY_ADDRESS), &id, 20);
    return true;
}

bool CBitcoinAddress::Set(const CScriptID& id)
{
    SetData(Params().Base58Prefix(CChainParams::SCRIPT_ADDRESS), &id, 20);
    return true;
}

bool CBitcoinAddress::Set(const CTxDestination& dest)
{
    return boost::apply_visitor(CBitcoinAddressVisitor(this), dest);
}

bool CBitcoinAddress::IsValid() const
{
    return IsValid(Params());
}

bool CBitcoinAddress::IsValid(const CChainParams& params) const
{
    bool fCorrectSize = vchData.size() == 20;
    bool fKnownVersion = vchVersion == params.Base58Prefix(CChainParams::PUBKEY_ADDRESS) ||
                         vchVersion == params.Base58Prefix(CChainParams::SCRIPT_ADDRESS);
    return fCorrectSize && fKnownVersion;
}

CTxDestination CBitcoinAddress::Get() const
{
    if (!IsValid())
        return CNoDestination();
    uint160 id;
    memcpy(&id, vchData.data(), 20);
    if (vchVersion == Params().Base58Prefix(CChainParams::PUBKEY_ADDRESS))
        return CKeyID(id);
    else if (vchVersion == Params().Base58Prefix(CChainParams::SCRIPT_ADDRESS))
        return CScriptID(id);
    else
        return CNoDestination();
}

bool CBitcoinAddress::GetKeyID(CKeyID& keyID) const
{
    if (!IsValid() || vchVersion != Params().Base58Prefix(CChainParams::PUBKEY_ADDRESS))
        return false;
    uint160 id;
    memcpy(&id, vchData.data(), 20);
    keyID = CKeyID(id);
    return true;
}

bool CBitcoinAddress::IsScript() const
{
    return IsValid() && vchVersion == Params().Base58Prefix(CChainParams::SCRIPT_ADDRESS);
}

void CBitcoinSecret::SetKey(const CKey& vchSecret)
{
    assert(vchSecret.IsValid());
    SetData(Params().Base58Prefix(CChainParams::SECRET_KEY), vchSecret.begin(), vchSecret.size());
    if (vchSecret.IsCompressed())
        vchData.push_back(1);
}

CKey CBitcoinSecret::GetKey()
{
    CKey ret;
    assert(vchData.size() >= 32);
    ret.Set(vchData.begin(), vchData.begin() + 32, vchData.size() > 32 && vchData[32] == 1);
    return ret;
}

bool CBitcoinSecret::IsValid() const
{
    bool fExpectedFormat = vchData.size() == 32 || (vchData.size() == 33 && vchData[32] == 1);
    bool fCorrectVersion = vchVersion == Params().Base58Prefix(CChainParams::SECRET_KEY);
    return fExpectedFormat && fCorrectVersion;
}

bool CBitcoinSecret::SetString(const char* pszSecret)
{
    return CBase58Data::SetString(pszSecret) && IsValid();
}

bool CBitcoinSecret::SetString(const std::string& strSecret)
{
    return SetString(strSecret.c_str());
}

std::string EncodeDestination(const CTxDestination& dest)
{
    CBitcoinAddress addr(dest);
    if (!addr.IsValid()) return "";
    return addr.ToString();
}

CTxDestination DecodeDestination(const std::string& str)
{
    return CBitcoinAddress(str).Get();
}

bool IsValidDestinationString(const std::string& str, const CChainParams& params)
{
    return CBitcoinAddress(str).IsValid(params);
}

bool IsValidDestinationString(const std::string& str)
{
    return CBitcoinAddress(str).IsValid();
}
<commit_msg>Remove unused GetKeyID and IsScript methods from CBitcoinAddress<commit_after>// Copyright (c) 2014-2018 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <base58.h>

#include <hash.h>
#include <uint256.h>

#include <assert.h>
#include <stdint.h>
#include <string.h>
#include <vector>
#include <string>
#include <boost/variant/apply_visitor.hpp>
#include <boost/variant/static_visitor.hpp>

/** All alphanumeric characters except for "0", "I", "O", and "l" */
/** All alphanumeric characters except for "0", "I", "O", and "l" */
static const char* pszBase58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";

bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch)
{
    // Skip leading spaces.
    while (*psz && isspace(*psz))
        psz++;
    // Skip and count leading '1's.
    int zeroes = 0;
    int length = 0;
    while (*psz == '1') {
        zeroes++;
        psz++;
    }
    // Allocate enough space in big-endian base256 representation.
    int size = strlen(psz) * 733 /1000 + 1; // log(58) / log(256), rounded up.
    std::vector<unsigned char> b256(size);
    // Process the characters.
    while (*psz && !isspace(*psz)) {
        // Decode base58 character
        const char* ch = strchr(pszBase58, *psz);
        if (ch == nullptr)
            return false;
        // Apply "b256 = b256 * 58 + ch".
        int carry = ch - pszBase58;
        int i = 0;
        for (std::vector<unsigned char>::reverse_iterator it = b256.rbegin(); (carry != 0 || i < length) && (it != b256.rend()); ++it, ++i) {
            carry += 58 * (*it);
            *it = carry % 256;
            carry /= 256;
        }
        assert(carry == 0);
        length = i;
        psz++;
    }
    // Skip trailing spaces.
    while (isspace(*psz))
        psz++;
    if (*psz != 0)
        return false;
    // Skip leading zeroes in b256.
    std::vector<unsigned char>::iterator it = b256.begin() + (size - length);
    while (it != b256.end() && *it == 0)
        it++;
    // Copy result into output vector.
    vch.reserve(zeroes + (b256.end() - it));
    vch.assign(zeroes, 0x00);
    while (it != b256.end())
        vch.push_back(*(it++));
    return true;
}

std::string EncodeBase58(const unsigned char* pbegin, const unsigned char* pend)
{
    // Skip & count leading zeroes.
    int zeroes = 0;
    int length = 0;
    while (pbegin != pend && *pbegin == 0) {
        pbegin++;
        zeroes++;
    }
    // Allocate enough space in big-endian base58 representation.
    int size = (pend - pbegin) * 138 / 100 + 1; // log(256) / log(58), rounded up.
    std::vector<unsigned char> b58(size);
    // Process the bytes.
    while (pbegin != pend) {
        int carry = *pbegin;
        int i = 0;
        // Apply "b58 = b58 * 256 + ch".
        for (std::vector<unsigned char>::reverse_iterator it = b58.rbegin(); (carry != 0 || i < length) && (it != b58.rend()); it++, i++) {
            carry += 256 * (*it);
            *it = carry % 58;
            carry /= 58;
        }

        assert(carry == 0);
        length = i;
        pbegin++;
    }
    // Skip leading zeroes in base58 result.
    std::vector<unsigned char>::iterator it = b58.begin() + (size - length);
    while (it != b58.end() && *it == 0)
        it++;
    // Translate the result into a string.
    std::string str;
    str.reserve(zeroes + (b58.end() - it));
    str.assign(zeroes, '1');
    while (it != b58.end())
        str += pszBase58[*(it++)];
    return str;
}

std::string EncodeBase58(const std::vector<unsigned char>& vch)
{
    return EncodeBase58(vch.data(), vch.data() + vch.size());
}

bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet)
{
    return DecodeBase58(str.c_str(), vchRet);
}

std::string EncodeBase58Check(const std::vector<unsigned char>& vchIn)
{
    // add 4-byte hash check to the end
    std::vector<unsigned char> vch(vchIn);
    uint256 hash = Hash(vch.begin(), vch.end());
    vch.insert(vch.end(), (unsigned char*)&hash, (unsigned char*)&hash + 4);
    return EncodeBase58(vch);
}

bool DecodeBase58Check(const char* psz, std::vector<unsigned char>& vchRet)
{
    if (!DecodeBase58(psz, vchRet) ||
        (vchRet.size() < 4)) {
        vchRet.clear();
        return false;
    }
    // re-calculate the checksum, ensure it matches the included 4-byte checksum
    uint256 hash = Hash(vchRet.begin(), vchRet.end() - 4);
    if (memcmp(&hash, &vchRet.end()[-4], 4) != 0) {
        vchRet.clear();
        return false;
    }
    vchRet.resize(vchRet.size() - 4);
    return true;
}

bool DecodeBase58Check(const std::string& str, std::vector<unsigned char>& vchRet)
{
    return DecodeBase58Check(str.c_str(), vchRet);
}

CBase58Data::CBase58Data()
{
    vchVersion.clear();
    vchData.clear();
}

void CBase58Data::SetData(const std::vector<unsigned char>& vchVersionIn, const void* pdata, size_t nSize)
{
    vchVersion = vchVersionIn;
    vchData.resize(nSize);
    if (!vchData.empty())
        memcpy(vchData.data(), pdata, nSize);
}

void CBase58Data::SetData(const std::vector<unsigned char>& vchVersionIn, const unsigned char* pbegin, const unsigned char* pend)
{
    SetData(vchVersionIn, (void*)pbegin, pend - pbegin);
}

bool CBase58Data::SetString(const char* psz, unsigned int nVersionBytes)
{
    std::vector<unsigned char> vchTemp;
    bool rc58 = DecodeBase58Check(psz, vchTemp);
    if ((!rc58) || (vchTemp.size() < nVersionBytes)) {
        vchData.clear();
        vchVersion.clear();
        return false;
    }
    vchVersion.assign(vchTemp.begin(), vchTemp.begin() + nVersionBytes);
    vchData.resize(vchTemp.size() - nVersionBytes);
    if (!vchData.empty())
        memcpy(vchData.data(), vchTemp.data() + nVersionBytes, vchData.size());
    memory_cleanse(vchTemp.data(), vchTemp.size());
    return true;
}

bool CBase58Data::SetString(const std::string& str)
{
    return SetString(str.c_str());
}

std::string CBase58Data::ToString() const
{
    std::vector<unsigned char> vch = vchVersion;
    vch.insert(vch.end(), vchData.begin(), vchData.end());
    return EncodeBase58Check(vch);
}

int CBase58Data::CompareTo(const CBase58Data& b58) const
{
    if (vchVersion < b58.vchVersion)
        return -1;
    if (vchVersion > b58.vchVersion)
        return 1;
    if (vchData < b58.vchData)
        return -1;
    if (vchData > b58.vchData)
        return 1;
    return 0;
}

namespace
{
/** base58-encoded Chaincoin addresses.
 * Public-key-hash-addresses have version 0 (or 111 testnet).
 * The data vector contains RIPEMD160(SHA256(pubkey)), where pubkey is the serialized public key.
 * Script-hash-addresses have version 5 (or 196 testnet).
 * The data vector contains RIPEMD160(SHA256(cscript)), where cscript is the serialized redemption script.
 */
class CBitcoinAddress : public CBase58Data {
public:
    bool Set(const CKeyID &id);
    bool Set(const CScriptID &id);
    bool Set(const CTxDestination &dest);
    bool IsValid() const;
    bool IsValid(const CChainParams &params) const;

    CBitcoinAddress() {}
    CBitcoinAddress(const CTxDestination &dest) { Set(dest); }
    CBitcoinAddress(const std::string& strAddress) { SetString(strAddress); }
    CBitcoinAddress(const char* pszAddress) { SetString(pszAddress); }

    CTxDestination Get() const;
};

class CBitcoinAddressVisitor : public boost::static_visitor<bool>
{
private:
    CBitcoinAddress* addr;

public:
    explicit CBitcoinAddressVisitor(CBitcoinAddress* addrIn) : addr(addrIn) {}

    bool operator()(const CKeyID& id) const { return addr->Set(id); }
    bool operator()(const CScriptID& id) const { return addr->Set(id); }
    bool operator()(const CNoDestination& no) const { return false; }
};

} // namespace

bool CBitcoinAddress::Set(const CKeyID& id)
{
    SetData(Params().Base58Prefix(CChainParams::PUBKEY_ADDRESS), &id, 20);
    return true;
}

bool CBitcoinAddress::Set(const CScriptID& id)
{
    SetData(Params().Base58Prefix(CChainParams::SCRIPT_ADDRESS), &id, 20);
    return true;
}

bool CBitcoinAddress::Set(const CTxDestination& dest)
{
    return boost::apply_visitor(CBitcoinAddressVisitor(this), dest);
}

bool CBitcoinAddress::IsValid() const
{
    return IsValid(Params());
}

bool CBitcoinAddress::IsValid(const CChainParams& params) const
{
    bool fCorrectSize = vchData.size() == 20;
    bool fKnownVersion = vchVersion == params.Base58Prefix(CChainParams::PUBKEY_ADDRESS) ||
                         vchVersion == params.Base58Prefix(CChainParams::SCRIPT_ADDRESS);
    return fCorrectSize && fKnownVersion;
}

CTxDestination CBitcoinAddress::Get() const
{
    if (!IsValid())
        return CNoDestination();
    uint160 id;
    memcpy(&id, vchData.data(), 20);
    if (vchVersion == Params().Base58Prefix(CChainParams::PUBKEY_ADDRESS))
        return CKeyID(id);
    else if (vchVersion == Params().Base58Prefix(CChainParams::SCRIPT_ADDRESS))
        return CScriptID(id);
    else
        return CNoDestination();
}

void CBitcoinSecret::SetKey(const CKey& vchSecret)
{
    assert(vchSecret.IsValid());
    SetData(Params().Base58Prefix(CChainParams::SECRET_KEY), vchSecret.begin(), vchSecret.size());
    if (vchSecret.IsCompressed())
        vchData.push_back(1);
}

CKey CBitcoinSecret::GetKey()
{
    CKey ret;
    assert(vchData.size() >= 32);
    ret.Set(vchData.begin(), vchData.begin() + 32, vchData.size() > 32 && vchData[32] == 1);
    return ret;
}

bool CBitcoinSecret::IsValid() const
{
    bool fExpectedFormat = vchData.size() == 32 || (vchData.size() == 33 && vchData[32] == 1);
    bool fCorrectVersion = vchVersion == Params().Base58Prefix(CChainParams::SECRET_KEY);
    return fExpectedFormat && fCorrectVersion;
}

bool CBitcoinSecret::SetString(const char* pszSecret)
{
    return CBase58Data::SetString(pszSecret) && IsValid();
}

bool CBitcoinSecret::SetString(const std::string& strSecret)
{
    return SetString(strSecret.c_str());
}

std::string EncodeDestination(const CTxDestination& dest)
{
    CBitcoinAddress addr(dest);
    if (!addr.IsValid()) return "";
    return addr.ToString();
}

CTxDestination DecodeDestination(const std::string& str)
{
    return CBitcoinAddress(str).Get();
}

bool IsValidDestinationString(const std::string& str, const CChainParams& params)
{
    return CBitcoinAddress(str).IsValid(params);
}

bool IsValidDestinationString(const std::string& str)
{
    return CBitcoinAddress(str).IsValid();
}
<|endoftext|>


<commit_before>/*
 *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "call/version.h"

namespace webrtc {

// The timestamp is always in UTC.
const char* const kSourceTimestamp = "WebRTC source stamp 2021-01-09T04:01:32";

void LoadWebRTCVersionInRegister() {
  // Using volatile to instruct the compiler to not optimize `p` away even
  // if it looks unused.
  const char* volatile p = kSourceTimestamp;
  static_cast<void>(p);
}

}  // namespace webrtc
<commit_msg>Update WebRTC code version (2021-01-11T04:01:55).<commit_after>/*
 *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "call/version.h"

namespace webrtc {

// The timestamp is always in UTC.
const char* const kSourceTimestamp = "WebRTC source stamp 2021-01-11T04:01:55";

void LoadWebRTCVersionInRegister() {
  // Using volatile to instruct the compiler to not optimize `p` away even
  // if it looks unused.
  const char* volatile p = kSourceTimestamp;
  static_cast<void>(p);
}

}  // namespace webrtc
<|endoftext|>