<commit_before>#include "bitfinex.h"
#include "parameters.h"
#include "utils/base64.h"
#include "curl_fun.h"
#include "hex_str.hpp"

#include "jansson.h"
#include "openssl/sha.h"
#include "openssl/hmac.h"

#include <string.h>
#include <iostream>
#include <unistd.h>
#include <sstream>
#include <math.h>
#include <sys/time.h>

namespace Bitfinex {

quote_t getQuote(Parameters& params)
{
  bool GETRequest = true;
  json_t *root = getJsonFromUrl(params, "https://api.bitfinex.com/v1/ticker/btcusd", "", GETRequest);

  const char *quote = json_string_value(json_object_get(root, "bid"));
  double bidValue = quote ? std::stod(quote) : 0.0;

  quote = json_string_value(json_object_get(root, "ask"));
  double askValue = quote ? std::stod(quote) : 0.0;

  json_decref(root);
  return std::make_pair(bidValue, askValue);
}

double getAvail(Parameters& params, std::string currency)
{
  json_t* root = authRequest(params, "https://api.bitfinex.com/v1/balances", "balances", "");
  while (json_object_get(root, "message") != NULL)
  {
    sleep(1.0);
    *params.logFile << "<Bitfinex> Error with JSON: " << json_dumps(root, 0) << ". Retrying..." << std::endl;
    root = authRequest(params, "https://api.bitfinex.com/v1/balances", "balances", "");
  }
  size_t arraySize = json_array_size(root);
  double availability = 0.0;
  const char* returnedText;
  for (size_t i = 0; i < arraySize; i++)
  {
    std::string tmpType = json_string_value(json_object_get(json_array_get(root, i), "type"));
    std::string tmpCurrency = json_string_value(json_object_get(json_array_get(root, i), "currency"));
    if (tmpType.compare("trading") == 0 && tmpCurrency.compare(currency.c_str()) == 0)
    {
      returnedText = json_string_value(json_object_get(json_array_get(root, i), "amount"));
      if (returnedText != NULL)
      {
        availability = atof(returnedText);
      }
      else
      {
        *params.logFile << "<Bitfinex> Error with the credentials." << std::endl;
        availability = 0.0;
      }
    }
  }
  json_decref(root);
  return availability;
}

std::string sendLongOrder(Parameters& params, std::string direction, double quantity, double price)
{
  return sendOrder(params, direction, quantity, price);
}

std::string sendShortOrder(Parameters& params, std::string direction, double quantity, double price)
{
  return sendOrder(params, direction, quantity, price);
}

std::string sendOrder(Parameters& params, std::string direction, double quantity, double price)
{
  *params.logFile << "<Bitfinex> Trying to send a \"" << direction << "\" limit order: " << quantity << "@$" << price << "..." << std::endl;
  std::ostringstream oss;
  oss << "\"symbol\":\"btcusd\", \"amount\":\"" << quantity << "\", \"price\":\"" << price << "\", \"exchange\":\"bitfinex\", \"side\":\"" << direction << "\", \"type\":\"limit\"";
  std::string options = oss.str();
  json_t* root = authRequest(params, "https://api.bitfinex.com/v1/order/new", "order/new", options);
  auto orderId = std::to_string(json_integer_value(json_object_get(root, "order_id")));
  *params.logFile << "<Bitfinex> Done (order ID: " << orderId << ")\n" << std::endl;
  json_decref(root);
  return orderId;
}

bool isOrderComplete(Parameters& params, std::string orderId)
{
  if (orderId == "0") return true;

  auto options =  "\"order_id\":" + orderId;
  json_t* root = authRequest(params, "https://api.bitfinex.com/v1/order/status", "order/status", options);
  bool isComplete = json_is_false(json_object_get(root, "is_live"));
  json_decref(root);
  return isComplete;
}

double getActivePos(Parameters& params)
{
  json_t* root = authRequest(params, "https://api.bitfinex.com/v1/positions", "positions", "");
  double position;
  if (json_array_size(root) == 0)
  {
    *params.logFile << "<Bitfinex> WARNING: BTC position not available, return 0.0" << std::endl;
    position = 0.0;
  }
  else
  {
    position = atof(json_string_value(json_object_get(json_array_get(root, 0), "amount")));
  }
  json_decref(root);
  return position;
}

double getLimitPrice(Parameters& params, double volume, bool isBid)
{
  json_t* root;
  bool GETRequest = true;
  if (isBid)
  {
    root = json_object_get(getJsonFromUrl(params, "https://api.bitfinex.com/v1/book/btcusd", "", GETRequest), "bids");
  }
  else
  {
    root = json_object_get(getJsonFromUrl(params, "https://api.bitfinex.com/v1/book/btcusd", "", GETRequest), "asks");
  }
  *params.logFile << "<Bitfinex> Looking for a limit price to fill " << fabs(volume) << " BTC..." << std::endl;
  double tmpVol = 0.0;
  double p;
  double v;
  int i = 0;
    // loop on volume
  while (tmpVol < fabs(volume) * params.orderBookFactor)
  {
    p = atof(json_string_value(json_object_get(json_array_get(root, i), "price")));
    v = atof(json_string_value(json_object_get(json_array_get(root, i), "amount")));
    *params.logFile << "<Bitfinex> order book: " << v << "@$" << p << std::endl;
    tmpVol += v;
    i++;
  }
  double limPrice = 0.0;
  limPrice = atof(json_string_value(json_object_get(json_array_get(root, i-1), "price")));
  json_decref(root);
  return limPrice;
}

json_t* authRequest(Parameters& params, std::string url, std::string request, std::string options)
{
  struct timeval tv;
  gettimeofday(&tv, NULL);
  unsigned long long nonce = (tv.tv_sec * 1000.0) + (tv.tv_usec * 0.001) + 0.5;
  std::ostringstream oss;
  if (options.empty())
  {
    oss << "{\"request\":\"/v1/" << request << "\",\"nonce\":\"" << nonce << "\"}";
  }
  else
  {
    oss << "{\"request\":\"/v1/" << request << "\",\"nonce\":\"" << nonce << "\", " << options << "}";
  }
  std::string tmpPayload = base64_encode(reinterpret_cast<const uint8_t *>(oss.str().c_str()), oss.str().length());
  oss.clear();
  oss.str("");
  oss << "X-BFX-PAYLOAD:" << tmpPayload;
  std::string payload;
  payload = oss.str();
  oss.clear();
  oss.str("");
  // signature
  uint8_t *digest = HMAC(EVP_sha384(), params.bitfinexSecret.c_str(), params.bitfinexSecret.length(), (uint8_t *)tmpPayload.c_str(), tmpPayload.length(), NULL, NULL);

  oss << "X-BFX-SIGNATURE:" << hex_str(digest, digest + SHA384_DIGEST_LENGTH);
  struct curl_slist *headers = NULL;
  std::string api = "X-BFX-APIKEY:" + std::string(params.bitfinexApi);
  headers = curl_slist_append(headers, api.c_str());
  headers = curl_slist_append(headers, payload.c_str());
  headers = curl_slist_append(headers, oss.str().c_str());
  CURLcode resCurl;
  if (params.curl)
  {
    std::string readBuffer;
    curl_easy_setopt(params.curl, CURLOPT_POST, 1L);
    curl_easy_setopt(params.curl, CURLOPT_HTTPHEADER, headers);
    curl_easy_setopt(params.curl, CURLOPT_POSTFIELDS, "");
    curl_easy_setopt(params.curl, CURLOPT_SSL_VERIFYPEER, 0L);
    curl_easy_setopt(params.curl, CURLOPT_WRITEFUNCTION, WriteCallback);
    curl_easy_setopt(params.curl, CURLOPT_WRITEDATA, &readBuffer);
    curl_easy_setopt(params.curl, CURLOPT_URL, url.c_str());
    curl_easy_setopt(params.curl, CURLOPT_CONNECTTIMEOUT, 10L);
    resCurl = curl_easy_perform(params.curl);
    json_t* root;
    json_error_t error;
    while (resCurl != CURLE_OK)
    {
      *params.logFile << "<Bitfinex> Error with cURL. Retry in 2 sec..." << std::endl;
      sleep(2.0);
      readBuffer = "";
      resCurl = curl_easy_perform(params.curl);
    }
    root = json_loads(readBuffer.c_str(), 0, &error);
    while (!root)
    {
      *params.logFile << "<Bitfinex> Error with JSON:\n" << error.text << std::endl;
      *params.logFile << "<Bitfinex> Buffer:\n" << readBuffer.c_str() << std::endl;
      *params.logFile << "<Bitfinex> Retrying..." << std::endl;
      sleep(2.0);
      readBuffer = "";
      resCurl = curl_easy_perform(params.curl);
      while (resCurl != CURLE_OK)
      {
        *params.logFile << "<Bitfinex> Error with cURL. Retry in 2 sec..." << std::endl;
        sleep(2.0);
        readBuffer = "";
        resCurl = curl_easy_perform(params.curl);
      }
      root = json_loads(readBuffer.c_str(), 0, &error);
    }
    curl_slist_free_all(headers);
    curl_easy_reset(params.curl);
    return root;
  }
  else
  {
    *params.logFile << "<Bitfinex> Error with cURL init." << std::endl;
    return NULL;
  }
}

}
<commit_msg>Initial bitfinex switchover to restapi.<commit_after>#include "bitfinex.h"
#include "parameters.h"
#include "utils/restapi.h"
#include "utils/base64.h"
#include "curl_fun.h"
#include "hex_str.hpp"

#include "jansson.h"
#include "openssl/sha.h"
#include "openssl/hmac.h"
#include <iostream>
#include <unistd.h>
#include <sstream>
#include <math.h>
#include <sys/time.h>

namespace Bitfinex {

static RestApi& queryHandle(Parameters &params)
{
  static RestApi query ("https://api.bitfinex.com",
                        params.cacert.c_str(), *params.logFile);
  return query;
}

quote_t getQuote(Parameters& params)
{
  auto &exchange = queryHandle(params);
  json_t *root = exchange.getRequest("/v1/ticker/btcusd");

  const char *quote = json_string_value(json_object_get(root, "bid"));
  double bidValue = quote ? std::stod(quote) : 0.0;

  quote = json_string_value(json_object_get(root, "ask"));
  double askValue = quote ? std::stod(quote) : 0.0;

  json_decref(root);
  return std::make_pair(bidValue, askValue);
}

double getAvail(Parameters& params, std::string currency)
{
  json_t* root = authRequest(params, "https://api.bitfinex.com/v1/balances", "balances", "");
  while (json_object_get(root, "message") != NULL)
  {
    sleep(1.0);
    *params.logFile << "<Bitfinex> Error with JSON: " << json_dumps(root, 0) << ". Retrying..." << std::endl;
    root = authRequest(params, "https://api.bitfinex.com/v1/balances", "balances", "");
  }
  size_t arraySize = json_array_size(root);
  double availability = 0.0;
  const char* returnedText;
  for (size_t i = 0; i < arraySize; i++)
  {
    std::string tmpType = json_string_value(json_object_get(json_array_get(root, i), "type"));
    std::string tmpCurrency = json_string_value(json_object_get(json_array_get(root, i), "currency"));
    if (tmpType.compare("trading") == 0 && tmpCurrency.compare(currency.c_str()) == 0)
    {
      returnedText = json_string_value(json_object_get(json_array_get(root, i), "amount"));
      if (returnedText != NULL)
      {
        availability = atof(returnedText);
      }
      else
      {
        *params.logFile << "<Bitfinex> Error with the credentials." << std::endl;
        availability = 0.0;
      }
    }
  }
  json_decref(root);
  return availability;
}

std::string sendLongOrder(Parameters& params, std::string direction, double quantity, double price)
{
  return sendOrder(params, direction, quantity, price);
}

std::string sendShortOrder(Parameters& params, std::string direction, double quantity, double price)
{
  return sendOrder(params, direction, quantity, price);
}

std::string sendOrder(Parameters& params, std::string direction, double quantity, double price)
{
  *params.logFile << "<Bitfinex> Trying to send a \"" << direction << "\" limit order: " << quantity << "@$" << price << "..." << std::endl;
  std::ostringstream oss;
  oss << "\"symbol\":\"btcusd\", \"amount\":\"" << quantity << "\", \"price\":\"" << price << "\", \"exchange\":\"bitfinex\", \"side\":\"" << direction << "\", \"type\":\"limit\"";
  std::string options = oss.str();
  json_t* root = authRequest(params, "https://api.bitfinex.com/v1/order/new", "order/new", options);
  auto orderId = std::to_string(json_integer_value(json_object_get(root, "order_id")));
  *params.logFile << "<Bitfinex> Done (order ID: " << orderId << ")\n" << std::endl;
  json_decref(root);
  return orderId;
}

bool isOrderComplete(Parameters& params, std::string orderId)
{
  if (orderId == "0") return true;

  auto options =  "\"order_id\":" + orderId;
  json_t* root = authRequest(params, "https://api.bitfinex.com/v1/order/status", "order/status", options);
  bool isComplete = json_is_false(json_object_get(root, "is_live"));
  json_decref(root);
  return isComplete;
}

double getActivePos(Parameters& params)
{
  json_t* root = authRequest(params, "https://api.bitfinex.com/v1/positions", "positions", "");
  double position;
  if (json_array_size(root) == 0)
  {
    *params.logFile << "<Bitfinex> WARNING: BTC position not available, return 0.0" << std::endl;
    position = 0.0;
  }
  else
  {
    position = atof(json_string_value(json_object_get(json_array_get(root, 0), "amount")));
  }
  json_decref(root);
  return position;
}

double getLimitPrice(Parameters& params, double volume, bool isBid)
{
  auto &exchange  = queryHandle(params);
  json_t *root    = exchange.getRequest("/v1/book/btcusd");
  json_t *bidask  = json_object_get(root, isBid ? "bids" : "asks");

  *params.logFile << "<Bitfinex> Looking for a limit price to fill " << fabs(volume) << " BTC..." << std::endl;
  double tmpVol = 0.0;
  double p;
  double v;
  int i = 0;
    // loop on volume
  while (tmpVol < fabs(volume) * params.orderBookFactor)
  {
    p = atof(json_string_value(json_object_get(json_array_get(bidask, i), "price")));
    v = atof(json_string_value(json_object_get(json_array_get(bidask, i), "amount")));
    *params.logFile << "<Bitfinex> order book: " << v << "@$" << p << std::endl;
    tmpVol += v;
    i++;
  }
  double limPrice = atof(json_string_value(json_object_get(json_array_get(bidask, i-1), "price")));
  json_decref(root);
  return limPrice;
}

json_t* authRequest(Parameters& params, std::string url, std::string request, std::string options)
{
  struct timeval tv;
  gettimeofday(&tv, NULL);
  unsigned long long nonce = (tv.tv_sec * 1000.0) + (tv.tv_usec * 0.001) + 0.5;
  std::ostringstream oss;
  if (options.empty())
  {
    oss << "{\"request\":\"/v1/" << request << "\",\"nonce\":\"" << nonce << "\"}";
  }
  else
  {
    oss << "{\"request\":\"/v1/" << request << "\",\"nonce\":\"" << nonce << "\", " << options << "}";
  }
  std::string tmpPayload = base64_encode(reinterpret_cast<const uint8_t *>(oss.str().c_str()), oss.str().length());
  oss.clear();
  oss.str("");
  oss << "X-BFX-PAYLOAD:" << tmpPayload;
  std::string payload;
  payload = oss.str();
  oss.clear();
  oss.str("");
  // signature
  uint8_t *digest = HMAC(EVP_sha384(), params.bitfinexSecret.c_str(), params.bitfinexSecret.length(), (uint8_t *)tmpPayload.c_str(), tmpPayload.length(), NULL, NULL);

  oss << "X-BFX-SIGNATURE:" << hex_str(digest, digest + SHA384_DIGEST_LENGTH);
  struct curl_slist *headers = NULL;
  std::string api = "X-BFX-APIKEY:" + std::string(params.bitfinexApi);
  headers = curl_slist_append(headers, api.c_str());
  headers = curl_slist_append(headers, payload.c_str());
  headers = curl_slist_append(headers, oss.str().c_str());
  CURLcode resCurl;
  if (params.curl)
  {
    std::string readBuffer;
    curl_easy_setopt(params.curl, CURLOPT_POST, 1L);
    curl_easy_setopt(params.curl, CURLOPT_HTTPHEADER, headers);
    curl_easy_setopt(params.curl, CURLOPT_POSTFIELDS, "");
    curl_easy_setopt(params.curl, CURLOPT_SSL_VERIFYPEER, 0L);
    curl_easy_setopt(params.curl, CURLOPT_WRITEFUNCTION, WriteCallback);
    curl_easy_setopt(params.curl, CURLOPT_WRITEDATA, &readBuffer);
    curl_easy_setopt(params.curl, CURLOPT_URL, url.c_str());
    curl_easy_setopt(params.curl, CURLOPT_CONNECTTIMEOUT, 10L);
    resCurl = curl_easy_perform(params.curl);
    json_t* root;
    json_error_t error;
    while (resCurl != CURLE_OK)
    {
      *params.logFile << "<Bitfinex> Error with cURL. Retry in 2 sec..." << std::endl;
      sleep(2.0);
      readBuffer = "";
      resCurl = curl_easy_perform(params.curl);
    }
    root = json_loads(readBuffer.c_str(), 0, &error);
    while (!root)
    {
      *params.logFile << "<Bitfinex> Error with JSON:\n" << error.text << std::endl;
      *params.logFile << "<Bitfinex> Buffer:\n" << readBuffer.c_str() << std::endl;
      *params.logFile << "<Bitfinex> Retrying..." << std::endl;
      sleep(2.0);
      readBuffer = "";
      resCurl = curl_easy_perform(params.curl);
      while (resCurl != CURLE_OK)
      {
        *params.logFile << "<Bitfinex> Error with cURL. Retry in 2 sec..." << std::endl;
        sleep(2.0);
        readBuffer = "";
        resCurl = curl_easy_perform(params.curl);
      }
      root = json_loads(readBuffer.c_str(), 0, &error);
    }
    curl_slist_free_all(headers);
    curl_easy_reset(params.curl);
    return root;
  }
  else
  {
    *params.logFile << "<Bitfinex> Error with cURL init." << std::endl;
    return NULL;
  }
}

}
<|endoftext|>


<commit_before>/* bzflag
 * Copyright (c) 1993 - 2005 Tim Riker
 *
 * This package is free software;  you can redistribute it and/or
 * modify it under the terms of the license found in the file
 * named COPYING that should have accompanied this file.
 *
 * THIS PACKAGE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
 * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 */

#include "common.h"

// implementation header
#include "CacheManager.h"

// system headers
#include <string>
#include <vector>
#include <iostream>
#include <algorithm>
#include <stdio.h>
#include <stdlib.h>
#include <strings.h>
#include <time.h>
#include <sys/types.h>
#include <sys/stat.h>
#ifndef _WIN32
#  include <unistd.h>
#endif

// common headers
#include "md5.h"
#include "bzfio.h"
#include "TextUtils.h"
#include "FileManager.h"
#include "StateDatabase.h"
#include "DirectoryNames.h"

// function prototypes
static bool fileExists(const std::string& name);
static void removeDirs(const std::string& path);
static void removeNewlines(char* c);
static std::string partialEncoding(const std::string& string);
static bool compareUsedDate(const CacheManager::CacheRecord& a,
                            const CacheManager::CacheRecord& b);

CacheManager CACHEMGR;

CacheManager::CacheManager()
{
  indexName = getCacheDirName();
  indexName += "CacheIndex.txt";
  return;
}

CacheManager::~CacheManager()
{
  return;
}

bool CacheManager::isCacheFileType(const std::string name) const
{
  if (strncasecmp(name.c_str(), "http://", 7) == 0) {
    return true;
  }
  if (strncasecmp(name.c_str(), "ftp://", 6) == 0) {
    return true;
  }
  return false;
}

std::string CacheManager::getLocalName(const std::string name) const
{
  std::string local = "";
  if (strncasecmp(name.c_str(), "http://", 7) == 0) {
    local = getCacheDirName() + "http/";
    local += partialEncoding(name.substr(7));
  }
  else if (strncasecmp(name.c_str(), "ftp://", 6) == 0) {
    local = getCacheDirName() + "ftp/";
    local += partialEncoding(name.substr(6));
  }
#ifdef _WIN32
  std::replace(local.begin(), local.end(), '/', '\\');
#endif
  return local;
}

bool CacheManager::findURL(const std::string& url, CacheRecord& record)
{
  int pos = findRecord(url);
  if (pos >= 0) {
    CacheRecord* rec = &records[pos];
    rec->usedDate = time(NULL); // update the timestamp
    record = *rec;
    return true;
  }
  return false;
}

bool CacheManager::addFile(CacheRecord& record, const void* data)
{
  if (((data == NULL) && (record.size != 0)) || (record.url.size() <= 0)) {
    return false;
  }

  record.name = getLocalName(record.url);
  std::ostream* out = FILEMGR.createDataOutStream(record.name);
  if (out == NULL) {
    return false;
  }

  bool replacement = false;
  CacheRecord* rec = &record;

  int pos = findRecord(record.url);
  if (pos >= 0) {
    records[pos] = record;
    rec = &records[pos];
    replacement = true;
  }

  out->write((char*)data, rec->size);

  rec->usedDate = time(NULL); // update the timestamp

  MD5 md5;
  md5.update((unsigned char *)data, rec->size);
  md5.finalize();
  rec->key = md5.hexdigest();

  if (!replacement) {
    records.push_back(*rec);
  }

  delete out;
  return true;
}

int CacheManager::findRecord(const std::string& url)
{
  for (unsigned int i = 0; i < records.size(); i++) {
    CacheRecord* rec = &(records[i]);
    if (url == rec->url) {
      return i;
    }
  }
  return -1;
}

bool CacheManager::loadIndex()
{
  records.clear();

  FILE* file = fopen(indexName.c_str(), "r");
  if (file == NULL) {
    return false;
  }

  char buffer[1024];
  while (fgets(buffer, 1024, file) != NULL) {
    removeNewlines(buffer);
    if ((buffer[0] == '\0') || (buffer[0] == '#')) {
      continue;
    }

    CacheRecord rec;
    rec.url = buffer;
    rec.name = getLocalName(rec.url);

    if (fgets(buffer, 1024, file) == NULL) {
      break;
    } else {
      removeNewlines(buffer);
    }
    std::string line = buffer;
    std::vector<std::string> tokens = TextUtils::tokenize(line, " ");
    if (tokens.size() != 4) {
      DEBUG1("loadCacheIndex (bad line): %s\n", buffer);
      continue;
    }
    rec.size = strtoul(tokens[0].c_str(), NULL, 10);
    rec.date = strtoul(tokens[1].c_str(), NULL, 10);
    rec.usedDate = strtoul(tokens[2].c_str(), NULL, 10);
    rec.key = tokens[3];
    if (fileExists(rec.name)) {
      records.push_back(rec);
    }
  }

  fclose(file);
  return true;
}

bool CacheManager::saveIndex()
{
  std::sort(records.begin(), records.end(), compareUsedDate);

  std::string tmpIndexName = indexName + ".tmp";

  FILE* file = fopen(tmpIndexName.c_str(), "w");
  if (file == NULL) {
    return false;
  }

  for (unsigned int i = 0; i < records.size(); i++) {
    const CacheRecord& rec = records[i];
    fprintf(file, "%s\n%u %u %u %s\n\n", rec.url.c_str(),
            rec.size, rec.date, rec.usedDate, rec.key.c_str());
  }

  fclose(file);

  return (rename(tmpIndexName.c_str(), indexName.c_str()) == 0);
}

void CacheManager::limitCacheSize()
{
  int maxSize = BZDB.evalInt("maxCacheMB") * 1024 * 1024;
  if (maxSize < 0) {
    maxSize = 0;
  }

  int currentSize = 0;
  for (unsigned int i = 0; i < records.size(); i++) {
    currentSize += records[i].size;
  }

  std::sort(records.begin(), records.end(), compareUsedDate);

  while ((currentSize > maxSize) && (records.size() > 0)) {
    CacheManager::CacheRecord& rec = records.back();
    currentSize -= rec.size;
    remove(rec.name.c_str());
    removeDirs(rec.name);
    records.pop_back();
  }

  return;
}

std::vector<CacheManager::CacheRecord> CacheManager::getCacheList() const
{
  return records;
}

static bool fileExists (const std::string& name)
{
  struct stat buf;
#ifndef _WIN32
  return (stat(name.c_str(), &buf) == 0);
#else
  // Windows sucks yet again, if there is a trailing  "\"
  // at the end of the filename, _stat will return -1.
  std::string dirname = name;
  while (dirname.find_last_of('\\') == (dirname.size() - 1)) {
    dirname.resize(dirname.size() - 1);
  }
  return (_stat(dirname.c_str(), (struct _stat *) &buf) == 0);
#endif
}

static void removeDirs(const std::string& path)
{
  unsigned int minLen = getConfigDirName().size();
  std::string tmp = path;
  while (tmp.size() > minLen) {
    unsigned int i = tmp.find_last_of('/');
    tmp = tmp.substr(0, i);
    if (remove(tmp.c_str()) != 0) {
      break;
    }
  }
  return;
}

static void removeNewlines(char* c)
{
  while (*c != '\0') {
    if ((*c == '\n') || (*c == '\r')) {
      *c = '\0';
    }
    c++;
  }
  return;
}

static std::string partialEncoding(const std::string& string)
{
  // URL encoding removes the '/' and '.', which is
  // not acceptable. It is nice to have the directory
  // structure, and to be able to point and click your
  // way through it to view ".png"s.
  std::string tmp;
  char hex[5];
  for (unsigned int i = 0; i < string.size(); i++) {
    const char c = string[i];
    if (TextUtils::isWhitespace(c)) {
      tmp += "%20";
    }
    else if ((c == '%') || (c == '*') || (c == '?') ||
             (c == ':') || (c == '"') || (c == '\\')) {
      tmp += '%';
      sprintf(hex, "%-2.2X", c);
      tmp += hex;
    }
    else {
      tmp += c;
    }
  }
  return tmp;
}

static bool compareUsedDate(const CacheManager::CacheRecord& a,
                            const CacheManager::CacheRecord& b)
{
  // oldest last
  return (a.usedDate > b.usedDate);
}

/*
 * Local Variables: ***
 * mode:C ***
 * tab-width: 8 ***
 * c-basic-offset: 2 ***
 * indent-tabs-mode: t ***
 * End: ***
 * ex: shiftwidth=2 tabstop=8
 */
<commit_msg>and another<commit_after>/* bzflag
 * Copyright (c) 1993 - 2005 Tim Riker
 *
 * This package is free software;  you can redistribute it and/or
 * modify it under the terms of the license found in the file
 * named COPYING that should have accompanied this file.
 *
 * THIS PACKAGE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
 * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 */

#include "common.h"

// implementation header
#include "CacheManager.h"

// system headers
#include <string>
#include <vector>
#include <iostream>
#include <algorithm>
#include <stdio.h>
#include <stdlib.h>
#include <strings.h>
#include <time.h>
#include <sys/types.h>
#include <sys/stat.h>
#ifndef _WIN32
#  include <unistd.h>
#endif

// common headers
#include "md5.h"
#include "bzfio.h"
#include "TextUtils.h"
#include "FileManager.h"
#include "StateDatabase.h"
#include "DirectoryNames.h"

// function prototypes
static bool fileExists(const std::string& name);
static void removeDirs(const std::string& path);
static void removeNewlines(char* c);
static std::string partialEncoding(const std::string& string);
static bool compareUsedDate(const CacheManager::CacheRecord& a,
                            const CacheManager::CacheRecord& b);

CacheManager CACHEMGR;

CacheManager::CacheManager()
{
  indexName = getCacheDirName();
  indexName += "CacheIndex.txt";
  return;
}

CacheManager::~CacheManager()
{
  return;
}

bool CacheManager::isCacheFileType(const std::string name) const
{
  if (strncasecmp(name.c_str(), "http://", 7) == 0) {
    return true;
  }
  if (strncasecmp(name.c_str(), "ftp://", 6) == 0) {
    return true;
  }
  return false;
}

std::string CacheManager::getLocalName(const std::string name) const
{
  std::string local = "";
  if (strncasecmp(name.c_str(), "http://", 7) == 0) {
    local = getCacheDirName() + "http/";
    local += partialEncoding(name.substr(7));
  }
  else if (strncasecmp(name.c_str(), "ftp://", 6) == 0) {
    local = getCacheDirName() + "ftp/";
    local += partialEncoding(name.substr(6));
  }
#ifdef _WIN32
  std::replace(local.begin(), local.end(), '/', '\\');
#endif
  return local;
}

bool CacheManager::findURL(const std::string& url, CacheRecord& record)
{
  int pos = findRecord(url);
  if (pos >= 0) {
    CacheRecord* rec = &records[pos];
    rec->usedDate = time(NULL); // update the timestamp
    record = *rec;
    return true;
  }
  return false;
}

bool CacheManager::addFile(CacheRecord& record, const void* data)
{
  if (((data == NULL) && (record.size != 0)) || (record.url.size() <= 0)) {
    return false;
  }

  record.name = getLocalName(record.url);
  std::ostream* out = FILEMGR.createDataOutStream(record.name);
  if (out == NULL) {
    return false;
  }

  bool replacement = false;
  CacheRecord* rec = &record;

  int pos = findRecord(record.url);
  if (pos >= 0) {
    records[pos] = record;
    rec = &records[pos];
    replacement = true;
  }

  out->write((char*)data, rec->size);

  rec->usedDate = time(NULL); // update the timestamp

  MD5 md5;
  md5.update((unsigned char *)data, rec->size);
  md5.finalize();
  rec->key = md5.hexdigest();

  if (!replacement) {
    records.push_back(*rec);
  }

  delete out;
  return true;
}

int CacheManager::findRecord(const std::string& url)
{
  for (unsigned int i = 0; i < records.size(); i++) {
    CacheRecord* rec = &(records[i]);
    if (url == rec->url) {
      return i;
    }
  }
  return -1;
}

bool CacheManager::loadIndex()
{
  records.clear();

  FILE* file = fopen(indexName.c_str(), "r");
  if (file == NULL) {
    return false;
  }

  char buffer[1024];
  while (fgets(buffer, 1024, file) != NULL) {
    removeNewlines(buffer);
    if ((buffer[0] == '\0') || (buffer[0] == '#')) {
      continue;
    }

    CacheRecord rec;
    rec.url = buffer;
    rec.name = getLocalName(rec.url);

    if (fgets(buffer, 1024, file) == NULL) {
      break;
    } else {
      removeNewlines(buffer);
    }
    std::string line = buffer;
    std::vector<std::string> tokens = TextUtils::tokenize(line, " ");
    if (tokens.size() != 4) {
      DEBUG1("loadCacheIndex (bad line): %s\n", buffer);
      continue;
    }
    rec.size = strtoul(tokens[0].c_str(), NULL, 10);
    rec.date = strtoul(tokens[1].c_str(), NULL, 10);
    rec.usedDate = strtoul(tokens[2].c_str(), NULL, 10);
    rec.key = tokens[3];
    if (fileExists(rec.name)) {
      records.push_back(rec);
    }
  }

  fclose(file);
  return true;
}

bool CacheManager::saveIndex()
{
  std::sort(records.begin(), records.end(), compareUsedDate);

  std::string tmpIndexName = indexName + ".tmp";

  FILE* file = fopen(tmpIndexName.c_str(), "w");
  if (file == NULL) {
    return false;
  }

  for (unsigned int i = 0; i < records.size(); i++) {
    const CacheRecord& rec = records[i];
    fprintf(file, "%s\n%u %u %u %s\n\n", rec.url.c_str(),
            rec.size, rec.date, rec.usedDate, rec.key.c_str());
  }

  fclose(file);

  return (rename(tmpIndexName.c_str(), indexName.c_str()) == 0);
}

void CacheManager::limitCacheSize()
{
  int maxSize = BZDB.evalInt("maxCacheMB") * 1024 * 1024;
  if (maxSize < 0) {
    maxSize = 0;
  }

  int currentSize = 0;
  for (unsigned int i = 0; i < records.size(); i++) {
    currentSize += records[i].size;
  }

  std::sort(records.begin(), records.end(), compareUsedDate);

  while ((currentSize > maxSize) && (records.size() > 0)) {
    CacheManager::CacheRecord& rec = records.back();
    currentSize -= rec.size;
    remove(rec.name.c_str());
    removeDirs(rec.name);
    records.pop_back();
  }

  return;
}

std::vector<CacheManager::CacheRecord> CacheManager::getCacheList() const
{
  return records;
}

static bool fileExists (const std::string& name)
{
  struct stat buf;
#ifndef _WIN32
  return (stat(name.c_str(), &buf) == 0);
#else
  // Windows sucks yet again, if there is a trailing  "\"
  // at the end of the filename, _stat will return -1.
  std::string dirname = name;
  while (dirname.find_last_of('\\') == (dirname.size() - 1)) {
    dirname.resize(dirname.size() - 1);
  }
  return (_stat(dirname.c_str(), (struct _stat *) &buf) == 0);
#endif
}

static void removeDirs(const std::string& path)
{
  unsigned int minLen = getConfigDirName().size();
  std::string tmp = path;
  while (tmp.size() > minLen) {
#ifndef _WIN32
    unsigned int i = tmp.find_last_of('/');
#else
    unsigned int i = tmp.find_last_of('\\');
#endif
    tmp = tmp.substr(0, i);
    if (remove(tmp.c_str()) != 0) {
      break;
    }
  }
  return;
}

static void removeNewlines(char* c)
{
  while (*c != '\0') {
    if ((*c == '\n') || (*c == '\r')) {
      *c = '\0';
    }
    c++;
  }
  return;
}

static std::string partialEncoding(const std::string& string)
{
  // URL encoding removes the '/' and '.', which is
  // not acceptable. It is nice to have the directory
  // structure, and to be able to point and click your
  // way through it to view ".png"s.
  std::string tmp;
  char hex[5];
  for (unsigned int i = 0; i < string.size(); i++) {
    const char c = string[i];
    if (TextUtils::isWhitespace(c)) {
      tmp += "%20";
    }
    else if ((c == '%') || (c == '*') || (c == '?') ||
             (c == ':') || (c == '"') || (c == '\\')) {
      tmp += '%';
      sprintf(hex, "%-2.2X", c);
      tmp += hex;
    }
    else {
      tmp += c;
    }
  }
  return tmp;
}

static bool compareUsedDate(const CacheManager::CacheRecord& a,
                            const CacheManager::CacheRecord& b)
{
  // oldest last
  return (a.usedDate > b.usedDate);
}

/*
 * Local Variables: ***
 * mode:C ***
 * tab-width: 8 ***
 * c-basic-offset: 2 ***
 * indent-tabs-mode: t ***
 * End: ***
 * ex: shiftwidth=2 tabstop=8
 */
<|endoftext|>


<commit_before>// Copyright (c) 2009 Chris Pickel <sfiera@gmail.com>
//
// This file is part of libsfz, a free software project.  You can redistribute
// it and/or modify it under the terms of the MIT License.

#include "sfz/Bytes.hpp"

#include <algorithm>

using std::max;

namespace sfz {

namespace {

const size_t kDefaultBytesSize = 16;

}  // namespace

const size_t Bytes::npos = -1;
const size_t BytesPiece::npos = -1;

Bytes::Bytes()
    : _data(new uint8_t[kDefaultBytesSize]),
      _size(0),
      _capacity(kDefaultBytesSize) { }

Bytes::Bytes(const Bytes& bytes)
    : _data(new uint8_t[bytes._capacity]),
      _size(bytes._size),
      _capacity(bytes._capacity) {
    memcpy(_data.get(), bytes._data.get(), _size);
}

Bytes::Bytes(const BytesPiece& bytes)
    : _data(new uint8_t[max(bytes.size(), kDefaultBytesSize)]),
      _size(bytes.size()),
      _capacity(max(bytes.size(), kDefaultBytesSize)) {
    memcpy(_data.get(), bytes.data(), _size);
}

Bytes::Bytes(const uint8_t* data, size_t size)
    : _data(new uint8_t[max(size, kDefaultBytesSize)]),
      _size(size),
      _capacity(max(size, kDefaultBytesSize)) {
    memcpy(_data.get(), data, size);
}

Bytes::Bytes(WriteItem item)
    : _data(new uint8_t[kDefaultBytesSize]),
      _size(0),
      _capacity(kDefaultBytesSize) {
    item.write_to(this);
}

Bytes::Bytes(size_t num, uint8_t byte)
    : _data(new uint8_t[max(num, kDefaultBytesSize)]),
      _size(num),
      _capacity(max(num, kDefaultBytesSize)) {
    memset(_data.get(), byte, num);
}

Bytes::~Bytes() { }

const uint8_t* Bytes::data() const {
    return _data.get();
}

uint8_t* Bytes::mutable_data() const {
    return _data.get();
}

size_t Bytes::size() const {
    return _size;
}

void Bytes::append(const BytesPiece& bytes) {
    append(bytes.data(), bytes.size());
}

void Bytes::append(const uint8_t* data, size_t size) {
    reserve(size + _size);
    memcpy(_data.get() + _size, data, size);
    _size += size;
}

void Bytes::append(WriteItem item) {
    item.write_to(this);
}

void Bytes::append(size_t num, uint8_t byte) {
    reserve(num + _size);
    memset(_data.get() + _size, byte, num);
    _size += num;
}

void Bytes::assign(const BytesPiece& bytes) {
    assign(bytes.data(), bytes.size());
}

void Bytes::assign(const uint8_t* data, size_t size) {
    reserve(size);
    memcpy(_data.get(), data, size);
    _size = size;
}

void Bytes::assign(WriteItem item) {
    clear();
    item.write_to(this);
}

void Bytes::assign(size_t num, uint8_t byte) {
    reserve(num);
    memset(_data.get(), byte, num);
    _size = num;
}

uint8_t Bytes::at(size_t loc) const {
    if (loc >= _size) {
        abort();
    }
    return _data.get()[loc];
}

void Bytes::clear() {
    _size = 0;
}

bool Bytes::empty() const {
    return _size == 0;
}

void Bytes::reserve(size_t capacity) {
    if (_capacity < capacity) {
        size_t new_capacity = _capacity * 2;
        while (new_capacity < capacity) {
            new_capacity *= 2;
        }
        scoped_array<uint8_t> new_data(new uint8_t[new_capacity]);
        memcpy(new_data.get(), _data.get(), _size);
        _data.swap(&new_data);
        _capacity = new_capacity;
    }
}

void Bytes::resize(size_t size, uint8_t byte) {
    if (size < _size) {
        _size = size;
    } else {
        reserve(size);
        memset(_data.get() + _size, byte, size - _size);
    }
}

void Bytes::swap(Bytes* bytes) {
    _data.swap(&bytes->_data);
    std::swap(_size, bytes->_size);
    std::swap(_capacity, bytes->_capacity);
}

BytesPiece::BytesPiece()
    : _data(NULL),
      _size(0) { }

BytesPiece::BytesPiece(const Bytes& bytes)
    : _data(bytes.data()),
      _size(bytes.size()) { }

BytesPiece::BytesPiece(const char* data)
    : _data(reinterpret_cast<const uint8_t*>(data)),
      _size(strlen(data)) { }

BytesPiece::BytesPiece(const uint8_t* data, size_t size)
    : _data(data),
      _size(size) { }

const uint8_t* BytesPiece::data() const {
    return _data;
}

size_t BytesPiece::size() const {
    return _size;
}

uint8_t BytesPiece::at(size_t loc) const {
    if (loc >= _size) {
        abort();
    }
    return _data[loc];
}

bool BytesPiece::empty() const {
    return _size == 0;
}

BytesPiece BytesPiece::substr(size_t index) const {
    if (index > _size) {
        abort();
    }
    return BytesPiece(_data + index, _size - index);
}

BytesPiece BytesPiece::substr(size_t index, size_t size) const {
    if (index + size > _size) {
        abort();
    }
    return BytesPiece(_data + index, size);
}

void BytesPiece::shift(size_t size) {
    if (size > _size) {
        abort();
    }
    _data += size;
    _size -= size;
}

void BytesPiece::shift(uint8_t* data, size_t size) {
    shift(size);
    memcpy(data, _data - size, size);
}

// Equality operators.

bool operator==(const Bytes& lhs, const Bytes& rhs) {
    return BytesPiece(lhs) == BytesPiece(rhs);
}

bool operator!=(const Bytes& lhs, const Bytes& rhs) {
    return BytesPiece(lhs) != BytesPiece(rhs);
}

bool operator==(const BytesPiece& lhs, const BytesPiece& rhs) {
    return (lhs.size() == rhs.size())
        && (memcmp(lhs.data(), rhs.data(), lhs.size()) == 0);
}

bool operator!=(const BytesPiece& lhs, const BytesPiece& rhs) {
    return !(lhs == rhs);
}

}  // namespace sfz
<commit_msg>Bugfix: set Bytes::_size after resize().<commit_after>// Copyright (c) 2009 Chris Pickel <sfiera@gmail.com>
//
// This file is part of libsfz, a free software project.  You can redistribute
// it and/or modify it under the terms of the MIT License.

#include "sfz/Bytes.hpp"

#include <algorithm>

using std::max;

namespace sfz {

namespace {

const size_t kDefaultBytesSize = 16;

}  // namespace

const size_t Bytes::npos = -1;
const size_t BytesPiece::npos = -1;

Bytes::Bytes()
    : _data(new uint8_t[kDefaultBytesSize]),
      _size(0),
      _capacity(kDefaultBytesSize) { }

Bytes::Bytes(const Bytes& bytes)
    : _data(new uint8_t[bytes._capacity]),
      _size(bytes._size),
      _capacity(bytes._capacity) {
    memcpy(_data.get(), bytes._data.get(), _size);
}

Bytes::Bytes(const BytesPiece& bytes)
    : _data(new uint8_t[max(bytes.size(), kDefaultBytesSize)]),
      _size(bytes.size()),
      _capacity(max(bytes.size(), kDefaultBytesSize)) {
    memcpy(_data.get(), bytes.data(), _size);
}

Bytes::Bytes(const uint8_t* data, size_t size)
    : _data(new uint8_t[max(size, kDefaultBytesSize)]),
      _size(size),
      _capacity(max(size, kDefaultBytesSize)) {
    memcpy(_data.get(), data, size);
}

Bytes::Bytes(WriteItem item)
    : _data(new uint8_t[kDefaultBytesSize]),
      _size(0),
      _capacity(kDefaultBytesSize) {
    item.write_to(this);
}

Bytes::Bytes(size_t num, uint8_t byte)
    : _data(new uint8_t[max(num, kDefaultBytesSize)]),
      _size(num),
      _capacity(max(num, kDefaultBytesSize)) {
    memset(_data.get(), byte, num);
}

Bytes::~Bytes() { }

const uint8_t* Bytes::data() const {
    return _data.get();
}

uint8_t* Bytes::mutable_data() const {
    return _data.get();
}

size_t Bytes::size() const {
    return _size;
}

void Bytes::append(const BytesPiece& bytes) {
    append(bytes.data(), bytes.size());
}

void Bytes::append(const uint8_t* data, size_t size) {
    reserve(size + _size);
    memcpy(_data.get() + _size, data, size);
    _size += size;
}

void Bytes::append(WriteItem item) {
    item.write_to(this);
}

void Bytes::append(size_t num, uint8_t byte) {
    reserve(num + _size);
    memset(_data.get() + _size, byte, num);
    _size += num;
}

void Bytes::assign(const BytesPiece& bytes) {
    assign(bytes.data(), bytes.size());
}

void Bytes::assign(const uint8_t* data, size_t size) {
    reserve(size);
    memcpy(_data.get(), data, size);
    _size = size;
}

void Bytes::assign(WriteItem item) {
    clear();
    item.write_to(this);
}

void Bytes::assign(size_t num, uint8_t byte) {
    reserve(num);
    memset(_data.get(), byte, num);
    _size = num;
}

uint8_t Bytes::at(size_t loc) const {
    if (loc >= _size) {
        abort();
    }
    return _data.get()[loc];
}

void Bytes::clear() {
    _size = 0;
}

bool Bytes::empty() const {
    return _size == 0;
}

void Bytes::reserve(size_t capacity) {
    if (_capacity < capacity) {
        size_t new_capacity = _capacity * 2;
        while (new_capacity < capacity) {
            new_capacity *= 2;
        }
        scoped_array<uint8_t> new_data(new uint8_t[new_capacity]);
        memcpy(new_data.get(), _data.get(), _size);
        _data.swap(&new_data);
        _capacity = new_capacity;
    }
}

void Bytes::resize(size_t size, uint8_t byte) {
    if (size < _size) {
        _size = size;
    } else {
        reserve(size);
        memset(_data.get() + _size, byte, size - _size);
        _size = size;
    }
}

void Bytes::swap(Bytes* bytes) {
    _data.swap(&bytes->_data);
    std::swap(_size, bytes->_size);
    std::swap(_capacity, bytes->_capacity);
}

BytesPiece::BytesPiece()
    : _data(NULL),
      _size(0) { }

BytesPiece::BytesPiece(const Bytes& bytes)
    : _data(bytes.data()),
      _size(bytes.size()) { }

BytesPiece::BytesPiece(const char* data)
    : _data(reinterpret_cast<const uint8_t*>(data)),
      _size(strlen(data)) { }

BytesPiece::BytesPiece(const uint8_t* data, size_t size)
    : _data(data),
      _size(size) { }

const uint8_t* BytesPiece::data() const {
    return _data;
}

size_t BytesPiece::size() const {
    return _size;
}

uint8_t BytesPiece::at(size_t loc) const {
    if (loc >= _size) {
        abort();
    }
    return _data[loc];
}

bool BytesPiece::empty() const {
    return _size == 0;
}

BytesPiece BytesPiece::substr(size_t index) const {
    if (index > _size) {
        abort();
    }
    return BytesPiece(_data + index, _size - index);
}

BytesPiece BytesPiece::substr(size_t index, size_t size) const {
    if (index + size > _size) {
        abort();
    }
    return BytesPiece(_data + index, size);
}

void BytesPiece::shift(size_t size) {
    if (size > _size) {
        abort();
    }
    _data += size;
    _size -= size;
}

void BytesPiece::shift(uint8_t* data, size_t size) {
    shift(size);
    memcpy(data, _data - size, size);
}

// Equality operators.

bool operator==(const Bytes& lhs, const Bytes& rhs) {
    return BytesPiece(lhs) == BytesPiece(rhs);
}

bool operator!=(const Bytes& lhs, const Bytes& rhs) {
    return BytesPiece(lhs) != BytesPiece(rhs);
}

bool operator==(const BytesPiece& lhs, const BytesPiece& rhs) {
    return (lhs.size() == rhs.size())
        && (memcmp(lhs.data(), rhs.data(), lhs.size()) == 0);
}

bool operator!=(const BytesPiece& lhs, const BytesPiece& rhs) {
    return !(lhs == rhs);
}

}  // namespace sfz
<|endoftext|>


<commit_before>/*
 * Distributed memory pool in shared memory.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#include "dpool.hxx"
#include "dchunk.hxx"
#include "shm.hxx"

#include <inline/compiler.h>
#include <inline/poison.h>

#include <boost/interprocess/sync/interprocess_mutex.hpp>
#include <boost/interprocess/sync/scoped_lock.hpp>

#include <assert.h>

#if defined(__x86_64__) || defined(__PPC64__)
#define ALIGN 8
#define ALIGN_BITS 0x7
#else
#define ALIGN 4
#define ALIGN_BITS 0x3
#endif

struct dpool {
    struct shm *const shm;
    mutable boost::interprocess::interprocess_mutex mutex;

    /**
     * Counts the number of d_free() calls.  After a certain number of
     * calls, we assume that the pool is "fragmented" and the session
     * shall be duplicated to a new pool.
     */
    unsigned free_counter = 0;

    DpoolChunk first_chunk;

    explicit dpool(struct shm &_shm);
    ~dpool();
};

dpool::dpool(struct shm &_shm)
    :shm(&_shm),
     first_chunk(shm_page_size(shm) - sizeof(*this) +
                 sizeof(first_chunk.data))
{
    assert(shm_page_size(shm) >= sizeof(*this));

    list_init(&first_chunk.siblings);
}

dpool::~dpool()
{
    assert(shm != nullptr);

    DpoolChunk *chunk, *n;
    for (chunk = (DpoolChunk *)first_chunk.siblings.next;
         chunk != &first_chunk; chunk = n) {
        n = (DpoolChunk *)chunk->siblings.next;

        chunk->Destroy(*shm);
    }
}

static constexpr size_t
align_size(size_t size)
{
    return ((size - 1) | ALIGN_BITS) + 1;
}

struct dpool *
dpool_new(struct shm &shm)
{
    return NewFromShm<struct dpool>(&shm, 1, shm);
}

void
dpool_destroy(struct dpool *pool)
{
    assert(pool != nullptr);

    DeleteFromShm(pool->shm, pool);
}

bool
dpool_is_fragmented(const struct dpool &pool)
{
    return pool.free_counter >= 256;
}

void *
d_malloc(struct dpool *pool, size_t size)
    throw(std::bad_alloc)
{
    void *p;

    assert(pool != nullptr);
    assert(pool->shm != nullptr);

    size = align_size(size);

    /* we could theoretically allow larger allocations by using
       multiple consecutive chunks, but we don't implement that
       because our current use cases should not need to allocate such
       large structures */
    if (size > pool->first_chunk.GetTotalSize())
        throw std::bad_alloc();

    boost::interprocess::scoped_lock<boost::interprocess::interprocess_mutex> scoped_lock(pool->mutex);

    /* find a chunk with enough room */

    auto *chunk = &pool->first_chunk;
    do {
        p = chunk->Allocate(size);
        if (p != nullptr)
            return p;

        chunk = (DpoolChunk *)chunk->siblings.next;
    } while (chunk != &pool->first_chunk);

    /* none found; try to allocate a new chunk */

    assert(p == nullptr);

    chunk = DpoolChunk::New(*pool->shm, pool->first_chunk.siblings);
    if (chunk == nullptr)
        throw std::bad_alloc();

    p = chunk->Allocate(size);
    assert(p != nullptr);
    return p;
}

static DpoolChunk *
dpool_find_chunk(struct dpool &pool, const void *p)
{
    if (pool.first_chunk.Contains(p))
        return &pool.first_chunk;

    for (auto *chunk = (DpoolChunk *)pool.first_chunk.siblings.next;
         chunk != &pool.first_chunk;
         chunk = (DpoolChunk *)chunk->siblings.next) {
        if (chunk->Contains(p))
            return chunk;
    }

    return nullptr;
}

void
d_free(struct dpool *pool, const void *p)
{
    boost::interprocess::scoped_lock<boost::interprocess::interprocess_mutex> scoped_lock(pool->mutex);

    ++pool->free_counter;

    auto *chunk = dpool_find_chunk(*pool, p);
    assert(chunk != nullptr);

    chunk->Free(const_cast<void *>(p));

    if (chunk->IsEmpty() && chunk != &pool->first_chunk) {
        /* the chunk is completely empty; release it to the SHM
           object */
        list_remove(&chunk->siblings);
        chunk->Destroy(*pool->shm);
    }
}
<commit_msg>shm/dpool: remove obsolete alignment code<commit_after>/*
 * Distributed memory pool in shared memory.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#include "dpool.hxx"
#include "dchunk.hxx"
#include "shm.hxx"

#include <inline/compiler.h>
#include <inline/poison.h>

#include <boost/interprocess/sync/interprocess_mutex.hpp>
#include <boost/interprocess/sync/scoped_lock.hpp>

#include <assert.h>

struct dpool {
    struct shm *const shm;
    mutable boost::interprocess::interprocess_mutex mutex;

    /**
     * Counts the number of d_free() calls.  After a certain number of
     * calls, we assume that the pool is "fragmented" and the session
     * shall be duplicated to a new pool.
     */
    unsigned free_counter = 0;

    DpoolChunk first_chunk;

    explicit dpool(struct shm &_shm);
    ~dpool();
};

dpool::dpool(struct shm &_shm)
    :shm(&_shm),
     first_chunk(shm_page_size(shm) - sizeof(*this) +
                 sizeof(first_chunk.data))
{
    assert(shm_page_size(shm) >= sizeof(*this));

    list_init(&first_chunk.siblings);
}

dpool::~dpool()
{
    assert(shm != nullptr);

    DpoolChunk *chunk, *n;
    for (chunk = (DpoolChunk *)first_chunk.siblings.next;
         chunk != &first_chunk; chunk = n) {
        n = (DpoolChunk *)chunk->siblings.next;

        chunk->Destroy(*shm);
    }
}

struct dpool *
dpool_new(struct shm &shm)
{
    return NewFromShm<struct dpool>(&shm, 1, shm);
}

void
dpool_destroy(struct dpool *pool)
{
    assert(pool != nullptr);

    DeleteFromShm(pool->shm, pool);
}

bool
dpool_is_fragmented(const struct dpool &pool)
{
    return pool.free_counter >= 256;
}

void *
d_malloc(struct dpool *pool, size_t size)
    throw(std::bad_alloc)
{
    void *p;

    assert(pool != nullptr);
    assert(pool->shm != nullptr);

    /* we could theoretically allow larger allocations by using
       multiple consecutive chunks, but we don't implement that
       because our current use cases should not need to allocate such
       large structures */
    if (size > pool->first_chunk.GetTotalSize())
        throw std::bad_alloc();

    boost::interprocess::scoped_lock<boost::interprocess::interprocess_mutex> scoped_lock(pool->mutex);

    /* find a chunk with enough room */

    auto *chunk = &pool->first_chunk;
    do {
        p = chunk->Allocate(size);
        if (p != nullptr)
            return p;

        chunk = (DpoolChunk *)chunk->siblings.next;
    } while (chunk != &pool->first_chunk);

    /* none found; try to allocate a new chunk */

    assert(p == nullptr);

    chunk = DpoolChunk::New(*pool->shm, pool->first_chunk.siblings);
    if (chunk == nullptr)
        throw std::bad_alloc();

    p = chunk->Allocate(size);
    assert(p != nullptr);
    return p;
}

static DpoolChunk *
dpool_find_chunk(struct dpool &pool, const void *p)
{
    if (pool.first_chunk.Contains(p))
        return &pool.first_chunk;

    for (auto *chunk = (DpoolChunk *)pool.first_chunk.siblings.next;
         chunk != &pool.first_chunk;
         chunk = (DpoolChunk *)chunk->siblings.next) {
        if (chunk->Contains(p))
            return chunk;
    }

    return nullptr;
}

void
d_free(struct dpool *pool, const void *p)
{
    boost::interprocess::scoped_lock<boost::interprocess::interprocess_mutex> scoped_lock(pool->mutex);

    ++pool->free_counter;

    auto *chunk = dpool_find_chunk(*pool, p);
    assert(chunk != nullptr);

    chunk->Free(const_cast<void *>(p));

    if (chunk->IsEmpty() && chunk != &pool->first_chunk) {
        /* the chunk is completely empty; release it to the SHM
           object */
        list_remove(&chunk->siblings);
        chunk->Destroy(*pool->shm);
    }
}
<|endoftext|>


<commit_before>/*
 * dwialgos.cpp
 *
 *  Created on: Jun 18, 2012
 *      Author: schurade
 */
#include "enums.h"
#include "../algos/fmath.h"
#include "../algos/track.h"

#include <QtCore/QDebug>
#include <QtCore/QVector>
#include <QtGui/QVector3D>

#include "../thirdparty/newmat10/newmat.h"
#include "../thirdparty/newmat10/newmatap.h"

#include "../gui/widgets/datasetselectionwidget.h"

#include "mesh/tesselation.h"

#include "datasets/dataset3d.h"
#include "datasets/datasetbingham.h"
#include "datasets/datasetdwi.h"
#include "datasets/datasetfibers.h"
#include "datasets/datasetscalar.h"
#include "datasets/datasettensor.h"
#include "datasets/datasetsh.h"

#include "../algos/qball.h"
#include "../algos/bingham.h"
#include "dwialgos.h"

DWIAlgos::DWIAlgos()
{
}

DWIAlgos::~DWIAlgos()
{
}

DatasetSH* DWIAlgos::qBall( DatasetDWI* ds )
{
    qDebug() << "start calculating qBall";
    QVector<QVector3D> bvecs = ds->getBvecs();

    Matrix gradients( bvecs.size(), 3 );
    for ( int i = 0; i < bvecs.size(); ++i )
    {
        gradients( i + 1, 1 ) = bvecs.at( i ).x();
        gradients( i + 1, 2 ) = bvecs.at( i ).y();
        gradients( i + 1, 3 ) = bvecs.at( i ).z();
    }

    double lambda = 0.006;
    int order = 4;
    Matrix qBallBase = QBall::calcQBallBase( gradients, lambda, order );

    QVector<ColumnVector>* data = ds->getData();

    QVector<ColumnVector>* qBallVector = new QVector<ColumnVector>();

    for ( int i = 0; i < data->size(); ++i )
    {
        qBallVector->push_back( qBallBase * data->at( i ) );
    }

    DatasetSH* out = new DatasetSH( "Q-Ball", qBallVector, ds->getHeader() );
    out->properties()->set( FNPROP_FILENAME, "QBall" );
    out->properties()->set( FNPROP_NAME, "QBall" );
    out->properties()->set( FNPROP_CREATED_BY, FNALGO_QBALL );
    out->properties()->set( FNPROP_LOD, 2 );
    out->properties()->set( FNPROP_ORDER, order );
    out->properties()->set( FNPROP_RENDER_SLICE, 1 );
    out->properties()->set( FNPROP_SCALING, 1.0f );
    out->properties()->set( FNPROP_DIM, qBallVector->at( 0 ).Nrows() );
    out->properties()->set( FNPROP_DATATYPE, DT_FLOAT );

    qDebug() << "finished calculating qBall";

    return out;
}

DatasetSH* DWIAlgos::qBallSharp( DatasetDWI* ds, int order )
{
    QVector<ColumnVector>* qBallVector = QBall::sharpQBall( ds, order );
    qDebug() << "create dataset";
    DatasetSH* out = new DatasetSH( "Q-Ball", qBallVector, ds->getHeader() );
    out->properties()->set( FNPROP_FILENAME, "Q-Ball" );

    QString name = QString( "Qball_" + QString::number( order ) + "_" + ds->properties()->get( FNPROP_NAME ).toString() );
    out->properties()->set( FNPROP_NAME, name );
    out->properties()->set( FNPROP_CREATED_BY, FNALGO_QBALL );
    out->properties()->set( FNPROP_LOD, 2 );
    out->properties()->set( FNPROP_ORDER, order );
    out->properties()->set( FNPROP_RENDER_SLICE, 1 );
    out->properties()->set( FNPROP_SCALING, 1.0f );
    out->properties()->set( FNPROP_DIM, qBallVector->at( 0 ).Nrows() );
    out->properties()->set( FNPROP_DATATYPE, DT_FLOAT );

    qDebug() << "finished calculating qBall";

    return out;

}

DatasetTensor* DWIAlgos::tensorFit( DatasetDWI* ds )
{
    QVector<QVector3D> bvecs = ds->getBvecs();
    QVector<float> bvals = ds->getBvals();
    QVector<ColumnVector>* data = ds->getData();
    QVector<float>* b0Images = ds->getB0Data();

    QVector<Matrix>* tensors = FMath::fitTensors( data, b0Images, bvecs, bvals );

    DatasetTensor* out = new DatasetTensor( ds->properties()->get( FNPROP_FILENAME ).toString(), tensors, ds->getHeader() );
    out->properties()->set( FNPROP_FILENAME, "Tensor" );
    out->properties()->set( FNPROP_NAME, "Tensor" );
    out->properties()->set( FNPROP_CREATED_BY, FNALGO_TENSORFIT );
    out->properties()->set( FNPROP_DIM, 9 );
    out->properties()->set( FNPROP_DATATYPE, DT_FLOAT );
    return out;
}

DatasetScalar* DWIAlgos::calcFAFromDWI( DatasetDWI* ds )
{
    QVector<QVector3D> bvecs = ds->getBvecs();
    QVector<float> bvals = ds->getBvals();
    QVector<ColumnVector>* data = ds->getData();
    QVector<float>* b0Images = ds->getB0Data();

    QVector<Matrix>* tensors = FMath::fitTensors( data, b0Images, bvecs, bvals );

    QVector<float> fa;
    FMath::fa( tensors, fa );

    DatasetScalar* out = new DatasetScalar( "fa.nii.gz", fa, ds->getHeader() );
    out->properties()->set( FNPROP_FILENAME, "FA" );
    out->properties()->set( FNPROP_NAME, "FA" );
    out->properties()->set( FNPROP_CREATED_BY, FNALGO_FA );
    out->properties()->set( FNPROP_DIM, 1 );
    out->properties()->set( FNPROP_DATATYPE, DT_FLOAT );
    return out;
}

QList<Dataset*> DWIAlgos::calcEVFromDWI( DatasetDWI* ds )
{
    QVector<QVector3D> bvecs = ds->getBvecs();
    QVector<float> bvals = ds->getBvals();
    QVector<ColumnVector>* data = ds->getData();
    QVector<float>* b0Images = ds->getB0Data();

    QVector<Matrix>* tensors = FMath::fitTensors( data, b0Images, bvecs, bvals );

    int blockSize = tensors->size();

    QVector<QVector3D> evec1( blockSize );
    QVector<float> eval1( blockSize );

    QVector<QVector3D> evec2( blockSize );
    QVector<float> eval2( blockSize );

    QVector<QVector3D> evec3( blockSize );
    QVector<float> eval3( blockSize );

    FMath::evecs( tensors, evec1, eval2, evec1, eval2, evec3, eval3 );

    QList<Dataset*> l;

    Dataset3D* out = new Dataset3D( "evec1.nii.gz", evec1, ds->getHeader() );
    out->properties()->set( FNPROP_NAME, "evec 1" );
    out->properties()->set( FNPROP_CREATED_BY, FNALGO_EV );
    out->properties()->set( FNPROP_DIM, 3 );
    out->properties()->set( FNPROP_DATATYPE, DT_FLOAT );

    DatasetScalar* out2 = new DatasetScalar( "eval1.nii.gz", eval1, ds->getHeader() );
    out2->properties()->set( FNPROP_NAME, "eval 1" );
    out2->properties()->set( FNPROP_CREATED_BY, FNALGO_EV );
    out2->properties()->set( FNPROP_DIM, 1 );
    out2->properties()->set( FNPROP_DATATYPE, DT_FLOAT );

    l.push_back( out );
    l.push_back( out2 );

    return l;
}

DatasetScalar* DWIAlgos::calcFAFromTensor( DatasetTensor* ds )
{
    QVector<Matrix>* tensors = ds->getData();

    QVector<float> fa;
    FMath::fa( tensors, fa );

    DatasetScalar* out = new DatasetScalar( "fa.nii.gz", fa, ds->getHeader() );
    out->properties()->set( FNPROP_FILENAME, "FA" );
    out->properties()->set( FNPROP_NAME, "FA" );
    out->properties()->set( FNPROP_CREATED_BY, FNALGO_FA );
    out->properties()->set( FNPROP_DIM, 1 );
    out->properties()->set( FNPROP_DATATYPE, DT_FLOAT );
    return out;
}

QList<Dataset*> DWIAlgos::calcEVFromTensor( DatasetTensor* ds )
{
    QVector<Matrix>* tensors = ds->getData();

    int blockSize = tensors->size();

    QVector<QVector3D> evec1( blockSize );
    QVector<float> eval1( blockSize );

    QVector<QVector3D> evec2( blockSize );
    QVector<float> eval2( blockSize );

    QVector<QVector3D> evec3( blockSize );
    QVector<float> eval3( blockSize );

    FMath::evecs( tensors, evec1, eval2, evec1, eval2, evec3, eval3 );

    QList<Dataset*> l;

    Dataset3D* out = new Dataset3D( "evec1.nii.gz", evec1, ds->getHeader() );
    out->properties()->set( FNPROP_NAME, "evec 1" );
    out->properties()->set( FNPROP_CREATED_BY, FNALGO_EV );
    out->properties()->set( FNPROP_DIM, 3 );
    out->properties()->set( FNPROP_DATATYPE, DT_FLOAT );

    DatasetScalar* out2 = new DatasetScalar( "eval1.nii.gz", eval1, ds->getHeader() );
    out2->properties()->set( FNPROP_NAME, "eval 1" );
    out2->properties()->set( FNPROP_CREATED_BY, FNALGO_EV );
    out2->properties()->set( FNPROP_DIM, 1 );
    out2->properties()->set( FNPROP_DATATYPE, DT_FLOAT );

    l.push_back( out );
    l.push_back( out2 );

    return l;
}

QList<Dataset*> DWIAlgos::fitBingham( DatasetSH* ds )
{
    int depth = 5;
    int neighbourhood = 3;
    int n_peaks = 3;

    QList<Dataset*> l= Bingham::calc_bingham( ds, depth, neighbourhood, n_peaks );
    if ( l.size() > 0 )
    {
        l[0]->properties()->set( FNPROP_NAME, "bingham" );
        l[0]->properties()->set( FNPROP_CREATED_BY, FNALGO_BINGHAM );
        l[0]->properties()->set( FNPROP_DIM, 42 );
        l[0]->properties()->set( FNPROP_DATATYPE, DT_FLOAT );
        //l[0]->properties()->set( "active", false );
    }
    return l;
}

QList<Dataset*> DWIAlgos::tensorTrack( DatasetTensor* ds )
{
    Track* tracker = new Track( ds );
    tracker->startTracking();

    QList<Dataset*> l;
    DatasetFibers* fibs = new DatasetFibers( tracker->getFibs(), tracker->getNumPoints(), tracker->getNumLines() );
    l.push_back( fibs );

    return l;
}

QList<Dataset*> DWIAlgos::bingham2Tensor( DatasetBingham* ds )
{
    QList<Dataset*> l= Bingham::bingham2Tensor( ds );
    for ( int i = 0; i < l.size(); ++i )
    {
        l[i]->properties()->set( FNPROP_FILENAME, "Tensor" );
        l[i]->properties()->set( FNPROP_NAME, "DWI FROM BINGHAM " + QString::number( i ) );
        l[i]->properties()->set( FNPROP_CREATED_BY, FNALGO_BINGHAM_2_TENSOR );
        l[i]->properties()->set( FNPROP_DIM, 9 );
        l[i]->properties()->set( FNPROP_DATATYPE, DT_FLOAT );
    }
    return l;
}

QList<Dataset*> DWIAlgos::testAlgo( Dataset* ds, QList<Dataset*> &dsl )
{
    QVector< QPair<QString, FN_DATASET_TYPE> >filter;

    filter.push_back( QPair<QString, FN_DATASET_TYPE>( "a scalar dataset", FNDT_NIFTI_SCALAR ) );
    filter.push_back( QPair<QString, FN_DATASET_TYPE>( "another scalar dataset", FNDT_NIFTI_SCALAR ) );
    filter.push_back( QPair<QString, FN_DATASET_TYPE>( "a bingham dataset", FNDT_NIFTI_BINGHAM ) );
    filter.push_back( QPair<QString, FN_DATASET_TYPE>( "any nifti dataset", FNDT_NIFTI_ANY ) );

    DatasetSelectionWidget* dsw = new DatasetSelectionWidget( filter, dsl );
    dsw->show();
}
<commit_msg>calcevfromdwi returned the second ev<commit_after>/*
 * dwialgos.cpp
 *
 *  Created on: Jun 18, 2012
 *      Author: schurade
 */
#include "enums.h"
#include "../algos/fmath.h"
#include "../algos/track.h"

#include <QtCore/QDebug>
#include <QtCore/QVector>
#include <QtGui/QVector3D>

#include "../thirdparty/newmat10/newmat.h"
#include "../thirdparty/newmat10/newmatap.h"

#include "../gui/widgets/datasetselectionwidget.h"

#include "mesh/tesselation.h"

#include "datasets/dataset3d.h"
#include "datasets/datasetbingham.h"
#include "datasets/datasetdwi.h"
#include "datasets/datasetfibers.h"
#include "datasets/datasetscalar.h"
#include "datasets/datasettensor.h"
#include "datasets/datasetsh.h"

#include "../algos/qball.h"
#include "../algos/bingham.h"
#include "dwialgos.h"

DWIAlgos::DWIAlgos()
{
}

DWIAlgos::~DWIAlgos()
{
}

DatasetSH* DWIAlgos::qBall( DatasetDWI* ds )
{
    qDebug() << "start calculating qBall";
    QVector<QVector3D> bvecs = ds->getBvecs();

    Matrix gradients( bvecs.size(), 3 );
    for ( int i = 0; i < bvecs.size(); ++i )
    {
        gradients( i + 1, 1 ) = bvecs.at( i ).x();
        gradients( i + 1, 2 ) = bvecs.at( i ).y();
        gradients( i + 1, 3 ) = bvecs.at( i ).z();
    }

    double lambda = 0.006;
    int order = 4;
    Matrix qBallBase = QBall::calcQBallBase( gradients, lambda, order );

    QVector<ColumnVector>* data = ds->getData();

    QVector<ColumnVector>* qBallVector = new QVector<ColumnVector>();

    for ( int i = 0; i < data->size(); ++i )
    {
        qBallVector->push_back( qBallBase * data->at( i ) );
    }

    DatasetSH* out = new DatasetSH( "Q-Ball", qBallVector, ds->getHeader() );
    out->properties()->set( FNPROP_FILENAME, "QBall" );
    out->properties()->set( FNPROP_NAME, "QBall" );
    out->properties()->set( FNPROP_CREATED_BY, FNALGO_QBALL );
    out->properties()->set( FNPROP_LOD, 2 );
    out->properties()->set( FNPROP_ORDER, order );
    out->properties()->set( FNPROP_RENDER_SLICE, 1 );
    out->properties()->set( FNPROP_SCALING, 1.0f );
    out->properties()->set( FNPROP_DIM, qBallVector->at( 0 ).Nrows() );
    out->properties()->set( FNPROP_DATATYPE, DT_FLOAT );

    qDebug() << "finished calculating qBall";

    return out;
}

DatasetSH* DWIAlgos::qBallSharp( DatasetDWI* ds, int order )
{
    QVector<ColumnVector>* qBallVector = QBall::sharpQBall( ds, order );
    qDebug() << "create dataset";
    DatasetSH* out = new DatasetSH( "Q-Ball", qBallVector, ds->getHeader() );
    out->properties()->set( FNPROP_FILENAME, "Q-Ball" );

    QString name = QString( "Qball_" + QString::number( order ) + "_" + ds->properties()->get( FNPROP_NAME ).toString() );
    out->properties()->set( FNPROP_NAME, name );
    out->properties()->set( FNPROP_CREATED_BY, FNALGO_QBALL );
    out->properties()->set( FNPROP_LOD, 2 );
    out->properties()->set( FNPROP_ORDER, order );
    out->properties()->set( FNPROP_RENDER_SLICE, 1 );
    out->properties()->set( FNPROP_SCALING, 1.0f );
    out->properties()->set( FNPROP_DIM, qBallVector->at( 0 ).Nrows() );
    out->properties()->set( FNPROP_DATATYPE, DT_FLOAT );

    qDebug() << "finished calculating qBall";

    return out;

}

DatasetTensor* DWIAlgos::tensorFit( DatasetDWI* ds )
{
    QVector<QVector3D> bvecs = ds->getBvecs();
    QVector<float> bvals = ds->getBvals();
    QVector<ColumnVector>* data = ds->getData();
    QVector<float>* b0Images = ds->getB0Data();

    QVector<Matrix>* tensors = FMath::fitTensors( data, b0Images, bvecs, bvals );

    DatasetTensor* out = new DatasetTensor( ds->properties()->get( FNPROP_FILENAME ).toString(), tensors, ds->getHeader() );
    out->properties()->set( FNPROP_FILENAME, "Tensor" );
    out->properties()->set( FNPROP_NAME, "Tensor" );
    out->properties()->set( FNPROP_CREATED_BY, FNALGO_TENSORFIT );
    out->properties()->set( FNPROP_DIM, 9 );
    out->properties()->set( FNPROP_DATATYPE, DT_FLOAT );
    return out;
}

DatasetScalar* DWIAlgos::calcFAFromDWI( DatasetDWI* ds )
{
    QVector<QVector3D> bvecs = ds->getBvecs();
    QVector<float> bvals = ds->getBvals();
    QVector<ColumnVector>* data = ds->getData();
    QVector<float>* b0Images = ds->getB0Data();

    QVector<Matrix>* tensors = FMath::fitTensors( data, b0Images, bvecs, bvals );

    QVector<float> fa;
    FMath::fa( tensors, fa );

    DatasetScalar* out = new DatasetScalar( "fa.nii.gz", fa, ds->getHeader() );
    out->properties()->set( FNPROP_FILENAME, "FA" );
    out->properties()->set( FNPROP_NAME, "FA" );
    out->properties()->set( FNPROP_CREATED_BY, FNALGO_FA );
    out->properties()->set( FNPROP_DIM, 1 );
    out->properties()->set( FNPROP_DATATYPE, DT_FLOAT );
    return out;
}

QList<Dataset*> DWIAlgos::calcEVFromDWI( DatasetDWI* ds )
{
    QVector<QVector3D> bvecs = ds->getBvecs();
    QVector<float> bvals = ds->getBvals();
    QVector<ColumnVector>* data = ds->getData();
    QVector<float>* b0Images = ds->getB0Data();

    QVector<Matrix>* tensors = FMath::fitTensors( data, b0Images, bvecs, bvals );

    int blockSize = tensors->size();

    QVector<QVector3D> evec1( blockSize );
    QVector<float> eval1( blockSize );

    QVector<QVector3D> evec2( blockSize );
    QVector<float> eval2( blockSize );

    QVector<QVector3D> evec3( blockSize );
    QVector<float> eval3( blockSize );

    FMath::evecs( tensors, evec1, eval1, evec2, eval2, evec3, eval3 );

    QList<Dataset*> l;

    Dataset3D* out = new Dataset3D( "evec1.nii.gz", evec1, ds->getHeader() );
    out->properties()->set( FNPROP_NAME, "evec 1" );
    out->properties()->set( FNPROP_CREATED_BY, FNALGO_EV );
    out->properties()->set( FNPROP_DIM, 3 );
    out->properties()->set( FNPROP_DATATYPE, DT_FLOAT );

    DatasetScalar* out2 = new DatasetScalar( "eval1.nii.gz", eval1, ds->getHeader() );
    out2->properties()->set( FNPROP_NAME, "eval 1" );
    out2->properties()->set( FNPROP_CREATED_BY, FNALGO_EV );
    out2->properties()->set( FNPROP_DIM, 1 );
    out2->properties()->set( FNPROP_DATATYPE, DT_FLOAT );

    l.push_back( out );
    l.push_back( out2 );

    return l;
}

DatasetScalar* DWIAlgos::calcFAFromTensor( DatasetTensor* ds )
{
    QVector<Matrix>* tensors = ds->getData();

    QVector<float> fa;
    FMath::fa( tensors, fa );

    DatasetScalar* out = new DatasetScalar( "fa.nii.gz", fa, ds->getHeader() );
    out->properties()->set( FNPROP_FILENAME, "FA" );
    out->properties()->set( FNPROP_NAME, "FA" );
    out->properties()->set( FNPROP_CREATED_BY, FNALGO_FA );
    out->properties()->set( FNPROP_DIM, 1 );
    out->properties()->set( FNPROP_DATATYPE, DT_FLOAT );
    return out;
}

QList<Dataset*> DWIAlgos::calcEVFromTensor( DatasetTensor* ds )
{
    QVector<Matrix>* tensors = ds->getData();

    int blockSize = tensors->size();

    QVector<QVector3D> evec1( blockSize );
    QVector<float> eval1( blockSize );

    QVector<QVector3D> evec2( blockSize );
    QVector<float> eval2( blockSize );

    QVector<QVector3D> evec3( blockSize );
    QVector<float> eval3( blockSize );

    FMath::evecs( tensors, evec1, eval2, evec1, eval2, evec3, eval3 );

    QList<Dataset*> l;

    Dataset3D* out = new Dataset3D( "evec1.nii.gz", evec1, ds->getHeader() );
    out->properties()->set( FNPROP_NAME, "evec 1" );
    out->properties()->set( FNPROP_CREATED_BY, FNALGO_EV );
    out->properties()->set( FNPROP_DIM, 3 );
    out->properties()->set( FNPROP_DATATYPE, DT_FLOAT );

    DatasetScalar* out2 = new DatasetScalar( "eval1.nii.gz", eval1, ds->getHeader() );
    out2->properties()->set( FNPROP_NAME, "eval 1" );
    out2->properties()->set( FNPROP_CREATED_BY, FNALGO_EV );
    out2->properties()->set( FNPROP_DIM, 1 );
    out2->properties()->set( FNPROP_DATATYPE, DT_FLOAT );

    l.push_back( out );
    l.push_back( out2 );

    return l;
}

QList<Dataset*> DWIAlgos::fitBingham( DatasetSH* ds )
{
    int depth = 5;
    int neighbourhood = 3;
    int n_peaks = 3;

    QList<Dataset*> l= Bingham::calc_bingham( ds, depth, neighbourhood, n_peaks );
    if ( l.size() > 0 )
    {
        l[0]->properties()->set( FNPROP_NAME, "bingham" );
        l[0]->properties()->set( FNPROP_CREATED_BY, FNALGO_BINGHAM );
        l[0]->properties()->set( FNPROP_DIM, 42 );
        l[0]->properties()->set( FNPROP_DATATYPE, DT_FLOAT );
        //l[0]->properties()->set( "active", false );
    }
    return l;
}

QList<Dataset*> DWIAlgos::tensorTrack( DatasetTensor* ds )
{
    Track* tracker = new Track( ds );
    tracker->startTracking();

    QList<Dataset*> l;
    DatasetFibers* fibs = new DatasetFibers( tracker->getFibs(), tracker->getNumPoints(), tracker->getNumLines() );
    l.push_back( fibs );

    return l;
}

QList<Dataset*> DWIAlgos::bingham2Tensor( DatasetBingham* ds )
{
    QList<Dataset*> l= Bingham::bingham2Tensor( ds );
    for ( int i = 0; i < l.size(); ++i )
    {
        l[i]->properties()->set( FNPROP_FILENAME, "Tensor" );
        l[i]->properties()->set( FNPROP_NAME, "DWI FROM BINGHAM " + QString::number( i ) );
        l[i]->properties()->set( FNPROP_CREATED_BY, FNALGO_BINGHAM_2_TENSOR );
        l[i]->properties()->set( FNPROP_DIM, 9 );
        l[i]->properties()->set( FNPROP_DATATYPE, DT_FLOAT );
    }
    return l;
}

QList<Dataset*> DWIAlgos::testAlgo( Dataset* ds, QList<Dataset*> &dsl )
{
    QVector< QPair<QString, FN_DATASET_TYPE> >filter;

    filter.push_back( QPair<QString, FN_DATASET_TYPE>( "a scalar dataset", FNDT_NIFTI_SCALAR ) );
    filter.push_back( QPair<QString, FN_DATASET_TYPE>( "another scalar dataset", FNDT_NIFTI_SCALAR ) );
    filter.push_back( QPair<QString, FN_DATASET_TYPE>( "a bingham dataset", FNDT_NIFTI_BINGHAM ) );
    filter.push_back( QPair<QString, FN_DATASET_TYPE>( "any nifti dataset", FNDT_NIFTI_ANY ) );

    DatasetSelectionWidget* dsw = new DatasetSelectionWidget( filter, dsl );
    dsw->show();
}
<|endoftext|>


<commit_before>#include <debugger/impl.h>
#include <map>

#if defined(_WIN32)
#pragma execution_character_set("utf-8")
#endif

namespace vscode {
	static std::map<std::string_view, translator_t> translators = {
		{ "zh-cn",{
			{ "All Exceptions", "所有异常" },
			{ "Uncaught Exceptions", "未捕获异常" },
		} }
	};

	void debugger_impl::setlang(const std::string_view& locale) {
		auto it = translators.find(locale);
		translator_ = it == translators.end() ? &it->second : nullptr;
	}

	std::string_view debugger_impl::LANG(const std::string_view& text) {
		if (!translator_) return text;
		auto it = translator_->find(text);
		if (it == translator_->end()) return text;
		return it->second;
	}
}<commit_msg>修正一个错误<commit_after>#include <debugger/impl.h>
#include <map>

#if defined(_WIN32)
#pragma execution_character_set("utf-8")
#endif

namespace vscode {
	static std::map<std::string_view, translator_t> translators = {
		{ "zh-cn",{
			{ "All Exceptions", "所有异常" },
			{ "Uncaught Exceptions", "未捕获异常" },
		} }
	};

	void debugger_impl::setlang(const std::string_view& locale) {
		auto it = translators.find(locale);
		translator_ = it != translators.end() ? &it->second : nullptr;
	}

	std::string_view debugger_impl::LANG(const std::string_view& text) {
		if (!translator_) return text;
		auto it = translator_->find(text);
		if (it == translator_->end()) return text;
		return it->second;
	}
}<|endoftext|>


<commit_before>#include "deconstructor.hpp"
#include "traversal_finder.hpp"

#define debug

using namespace std;

namespace vg {
Deconstructor::Deconstructor(){

}
Deconstructor::~Deconstructor(){
}

/**
 * Takes in a vector of snarltraversals
 * returns their sequences as a vector<string>
 * returns a boolean hasRef
 * if a reference path is present, hasRef is set to true and the first
 * string in the vector is the reference allele
 * otherwise, hasRef is set to false and all strings are alt alleles.
 */
vector<int> Deconstructor::get_alleles(vcflib::Variant& v, const vector<SnarlTraversal>& travs, int ref_path_idx,
                                       char prev_char) {

    assert(ref_path_idx >=0 && ref_path_idx < travs.size());

    // map strings to allele numbers
    // (we are using the traversal finder in such a way that duplicate alleles can get returned
    // in order to be able to preserve the path names)
    map<string, int> allele_idx;
    size_t cur_alt = 1;

    // go from traversals number (offset in travs) to allele number
    vector<int> trav_to_allele(travs.size());

    // compute the allele as a string
    auto trav_to_string = [&](const SnarlTraversal& trav) {
        string allele;
        // we skip the snarl endpoints
        for (int j = 1; j < trav.visit_size() - 1; ++j) {
            const string& node_sequence = graph->get_sequence(graph->get_handle(trav.visit(j).node_id()));
            allele += trav.visit(j).backward() ? reverse_complement(node_sequence) : node_sequence;
        }
        return toUppercase(allele);
    };

    // set the reference allele
    string ref_allele = trav_to_string(travs.at(ref_path_idx));
    allele_idx[ref_allele] = 0;
    trav_to_allele[ref_path_idx] = 0;
    bool substitution = true;

    // set the other alleles (they can end up as 0 alleles too if their strings match the reference)
    for (int i = 0; i < travs.size(); ++i) {
        if (i != ref_path_idx) {
            string allele = trav_to_string(travs[i]);
            auto ai_it = allele_idx.find(allele);
            if (ai_it == allele_idx.end()) {
                // make a new allele for this string
                allele_idx[allele] = cur_alt;
                trav_to_allele.at(i) = cur_alt;
                ++cur_alt;
                substitution = substitution && allele.size() == ref_allele.size();
            } else {
                // allele string has been seen, map this traversal to it
                trav_to_allele.at(i) = ai_it->second;
            }
        }
    }

    // fill in the variant
    v.alleles.resize(allele_idx.size());
    assert(allele_idx.size() > 0);
    v.alt.resize(allele_idx.size() - 1);

    for (auto ai_pair : allele_idx) {
        string allele_string = substitution ? ai_pair.first : string(1, prev_char) + ai_pair.first;
        v.alleles[ai_pair.second] = allele_string;
        if (ai_pair.second > 0) {
            v.alt[ai_pair.second - 1] = allele_string;
        } else {
            v.ref = allele_string;
        }
    }

    // shift our variant back if it's an indel
    if (!substitution) {
        --v.position;
        assert(v.position >= 1);
    }

    v.updateAlleleIndexes();

    return trav_to_allele;
}

void Deconstructor::get_genotypes(vcflib::Variant& v, const vector<string>& names,
                                  const vector<int>& trav_to_allele) {
    map<string, int> name_to_allele;
    for (int i = 0; i < names.size(); ++i) {
        name_to_allele[names[i]] = trav_to_allele[i];
    }
    v.format.push_back("GT");
    for (auto& sample_name : sample_names) {
        auto na = name_to_allele.find(sample_name);
        if (na != name_to_allele.end()) {
            v.samples[sample_name]["GT"] = {std::to_string(na->second)};
        } else {
            v.samples[sample_name]["GT"] = {"."};
        }
    }
}

bool Deconstructor::deconstruct_site(const Snarl* snarl) {

    auto contents = snarl_manager->shallow_contents(snarl, *graph, false);
    if (contents.first.empty()) {
        // Nothing but the boundary nodes in this snarl
#ifdef debug
#pragma omp critical (cerr)
        cerr << "Skipping empty site " << pb2json(*snarl) << endl;
#endif
        return false;
    }

    // find every traversal that runs through a path in the graph
    pair<vector<SnarlTraversal>, vector<string>> named_travs;
    named_travs = path_trav_finder->find_named_traversals(*snarl);

    // pick out the traversal corresponding to a reference path, breaking ties consistently
    int ref_trav_idx = -1;
    for (int i = 0; i < named_travs.first.size(); ++i) {
        if (pindexes.count(named_travs.second.at(i)) &&
            (ref_trav_idx == -1 || named_travs.second[i] < named_travs.second[ref_trav_idx])) {
            ref_trav_idx = i;
        }
    }

    // there's no reference path through the snarl, so we can't make a variant
    // (todo: should we try to detect this before computing traversals?)
    if (ref_trav_idx == -1) {
#ifdef debug
#pragma omp critical (cerr)
        cerr << "Skipping site becuase no reference traversal was found " << pb2json(*snarl) << endl;
#endif
        return false;
    }

    // add in the exhaustive traversals
    if (!path_restricted) {
        // exhaustive traversal can't do all snarls
        if (!snarl->type() == ULTRABUBBLE) {
            return false;
        }
        if (!check_max_nodes(snarl)) {
#pragma omp critical (cerr)
            cerr << "Warning: Skipping site because it is too complex for exhaustive traversal enumeration: " << pb2json(*snarl) << endl << "         Consider using -e to traverse embedded paths" << endl;
            return false;
        }
        vector<SnarlTraversal> exhaustive_travs = explicit_exhaustive_traversals(snarl);
        // happens when there was a nested non-ultrabubble snarl
        if (exhaustive_travs.empty()) {
            return false;
        }
        named_travs.first.insert(named_travs.first.end(), exhaustive_travs.begin(), exhaustive_travs.end());
        for (int i = 0; i < exhaustive_travs.size(); ++i) {
            // dummy names so we can use the same code as the named path traversals above
            named_travs.second.push_back(" >>" + std::to_string(i));
        }
    }

    // there's not alt path through the snarl, so we can't make an interesting variant
    if (named_travs.first.size() < 2) {
#ifdef debug
#pragma omp critical (cerr)
        cerr << "Skipping site because to alt traversal was found " << pb2json(*snarl) << endl;
#endif
        return false;
    }

    vcflib::Variant v;
    v.setVariantCallFile(outvcf);
    v.quality = 23;

    // write variant's sequenceName (VCF contig)
    v.sequenceName = named_travs.second.at(ref_trav_idx);

    // Set position based on the lowest position in the snarl.
    pair<size_t, bool> pos_orientation_start = pindexes[v.sequenceName]->by_id[snarl->start().node_id()];
    pair<size_t, bool> pos_orientation_end = pindexes[v.sequenceName]->by_id[snarl->end().node_id()];
    bool use_start = pos_orientation_start.first < pos_orientation_end.first;
    size_t node_pos = (use_start ? pos_orientation_start.first : pos_orientation_end.first);
    v.position = node_pos + 1; // shift from 0-based to 1-based
    if (use_start) {
        v.position += graph->get_node(snarl->start().node_id())->sequence().length();
    } else {
        v.position += graph->get_node(snarl->end().node_id())->sequence().length();
    }

    // We need to shift back a position for indels.  Get the previous base from the graph:
    const Visit& prev_visit = use_start ? snarl->start() : snarl->end();
    int offset = 0;
    if (use_start != prev_visit.backward()) {
        offset = graph->get_length(graph->get_handle(prev_visit.node_id())) - 1;
    }
    char prev_char = ::toupper(graph->get_sequence(graph->get_handle(prev_visit.node_id()))[offset]);

    // Convert the snarl traversals to strings and add them to the variant
    vector<int> trav_to_allele = get_alleles(v, named_travs.first, ref_trav_idx, prev_char);

    // Fill in the genotypes
    if (path_restricted) {
        get_genotypes(v, named_travs.second, trav_to_allele);
    }

#pragma omp critical (cout)
    {
        cout << v << endl;
    }

    return true;
}

/**
 * Convenience wrapper function for deconstruction of multiple paths.
 */
void Deconstructor::deconstruct(vector<string> ref_paths, vg::VG* graph, SnarlManager* snarl_manager,
                                bool path_restricted_traversals) {

    this->graph = graph;
    this->snarl_manager = snarl_manager;
    this->path_restricted = path_restricted_traversals;

    // Create path index for the contig if we don't have one.
    for (auto& refpath : ref_paths) {
        if (pindexes.find(refpath) == pindexes.end()){
            pindexes[refpath] = new PathIndex(*graph, refpath, false);
        }
    }

    // Keep track of the non-reference paths in the graph.  They'll be our sample names
    sample_names.clear();
    graph->for_each_path_handle([&](const path_handle_t& path_handle) {
            string path_name = graph->get_path_name(path_handle);
            if (!pindexes.count(path_name)) {
                sample_names.push_back(path_name);
            }
        });

    // print the VCF header
    stringstream stream;
    stream << "##fileformat=VCFv4.2" << endl;
    if (path_restricted) {
        stream << "##FORMAT=<ID=GT,Number=1,Type=String,Description=\"Genotype\">" << endl;
    }
    for(auto& refpath : ref_paths) {
        size_t path_len = 0;
        path_handle_t path_handle = graph->get_path_handle(refpath);
        for (handle_t handle : graph->scan_path(path_handle)) {
            path_len += graph->get_length(handle);
        }
        stream << "##contig=<ID=" << refpath << ",length=" << path_len << ">" << endl;
    }
    stream << "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT";
    if (path_restricted) {
        for (auto& sample_name : sample_names) {
            stream << "\t" << sample_name;
        }
    }
    stream << endl;

    string hstr = stream.str();
    assert(outvcf.openForOutput(hstr));
    cout << outvcf.header << endl;

    // create the traversal finder
    map<string, const Alignment*> reads_by_name;
    path_trav_finder = unique_ptr<PathRestrictedTraversalFinder>(new PathRestrictedTraversalFinder(*graph,
                                                                                *snarl_manager,
                                                                                reads_by_name,
                                                                                1,
                                                                                100,
                                                                                true));

    if (!path_restricted) {
        trav_finder = unique_ptr<TraversalFinder>(new ExhaustiveTraversalFinder(*graph,
                                                                                *snarl_manager,
                                                                                true));

    }

    // Do the top-level snarls in parallel
    snarl_manager->for_each_top_level_snarl_parallel([&](const Snarl* snarl) {
            vector<const Snarl*> todo(1, snarl);
            vector<const Snarl*> next;
            while (!todo.empty()) {
                for (auto next_snarl : todo) {
                    // if we can't make a variant from the snarl due to not finding
                    // paths through it, we try again on the children
                    // note: we may want to push the parallelism down a bit
                    if (!deconstruct_site(next_snarl)) {
                        const vector<const Snarl*>& children = snarl_manager->children_of(next_snarl);
                        next.insert(next.end(), children.begin(), children.end());
                    }
                }
                swap(todo, next);
                next.clear();
            }
        });
}

bool Deconstructor::check_max_nodes(const Snarl* snarl)  {
    unordered_set<Node*> nodeset = snarl_manager->deep_contents(snarl, *graph, false).first;
    int node_count = 0;
    for (auto node : nodeset) {
        if (graph->start_degree(node) > 1 || graph->end_degree(node) > 1) {
            ++node_count;
            if (node_count > max_nodes_for_exhaustive) {
                return false;
            }
        }
    }
    return true;
};

vector<SnarlTraversal> Deconstructor::explicit_exhaustive_traversals(const Snarl* snarl){
    vector<SnarlTraversal> out_travs;
    bool ultra_all_the_way_down = true;
    function<void(const SnarlTraversal&, const Snarl&)> extend_trav =
        [&](const SnarlTraversal& trav, const Snarl& nested_snarl) {
        // exhaustive traversal finder is limited.  if we find something
        // that's not an ultrabubble, not much we can do
        if (nested_snarl.type() != ULTRABUBBLE) {
            ultra_all_the_way_down = false;
            return;
        }
        vector<SnarlTraversal> nested_travs = trav_finder->find_traversals(nested_snarl);
        for (auto& nested_trav : nested_travs) {
            SnarlTraversal extended_trav = trav;
            bool is_explicit = true;
            for (int i = 0; i < nested_trav.visit_size(); ++i) {
                if (nested_trav.visit(i).node_id() != 0) {
                    Visit* visit = extended_trav.add_visit();
                    *visit = nested_trav.visit(i);
                } else {
                    extend_trav(extended_trav, nested_trav.visit(i).snarl());
                    is_explicit = false;
                }
            }
            if (is_explicit) {
                out_travs.push_back(extended_trav);
            }
        }
    };
    SnarlTraversal trav;
    extend_trav(trav, *snarl);
    if (!ultra_all_the_way_down) {
        out_travs.clear();
    }
    return out_travs;
}

}

<commit_msg>turn off debug<commit_after>#include "deconstructor.hpp"
#include "traversal_finder.hpp"

//#define debug

using namespace std;

namespace vg {
Deconstructor::Deconstructor(){

}
Deconstructor::~Deconstructor(){
}

/**
 * Takes in a vector of snarltraversals
 * returns their sequences as a vector<string>
 * returns a boolean hasRef
 * if a reference path is present, hasRef is set to true and the first
 * string in the vector is the reference allele
 * otherwise, hasRef is set to false and all strings are alt alleles.
 */
vector<int> Deconstructor::get_alleles(vcflib::Variant& v, const vector<SnarlTraversal>& travs, int ref_path_idx,
                                       char prev_char) {

    assert(ref_path_idx >=0 && ref_path_idx < travs.size());

    // map strings to allele numbers
    // (we are using the traversal finder in such a way that duplicate alleles can get returned
    // in order to be able to preserve the path names)
    map<string, int> allele_idx;
    size_t cur_alt = 1;

    // go from traversals number (offset in travs) to allele number
    vector<int> trav_to_allele(travs.size());

    // compute the allele as a string
    auto trav_to_string = [&](const SnarlTraversal& trav) {
        string allele;
        // we skip the snarl endpoints
        for (int j = 1; j < trav.visit_size() - 1; ++j) {
            const string& node_sequence = graph->get_sequence(graph->get_handle(trav.visit(j).node_id()));
            allele += trav.visit(j).backward() ? reverse_complement(node_sequence) : node_sequence;
        }
        return toUppercase(allele);
    };

    // set the reference allele
    string ref_allele = trav_to_string(travs.at(ref_path_idx));
    allele_idx[ref_allele] = 0;
    trav_to_allele[ref_path_idx] = 0;
    bool substitution = true;

    // set the other alleles (they can end up as 0 alleles too if their strings match the reference)
    for (int i = 0; i < travs.size(); ++i) {
        if (i != ref_path_idx) {
            string allele = trav_to_string(travs[i]);
            auto ai_it = allele_idx.find(allele);
            if (ai_it == allele_idx.end()) {
                // make a new allele for this string
                allele_idx[allele] = cur_alt;
                trav_to_allele.at(i) = cur_alt;
                ++cur_alt;
                substitution = substitution && allele.size() == ref_allele.size();
            } else {
                // allele string has been seen, map this traversal to it
                trav_to_allele.at(i) = ai_it->second;
            }
        }
    }

    // fill in the variant
    v.alleles.resize(allele_idx.size());
    assert(allele_idx.size() > 0);
    v.alt.resize(allele_idx.size() - 1);

    for (auto ai_pair : allele_idx) {
        string allele_string = substitution ? ai_pair.first : string(1, prev_char) + ai_pair.first;
        v.alleles[ai_pair.second] = allele_string;
        if (ai_pair.second > 0) {
            v.alt[ai_pair.second - 1] = allele_string;
        } else {
            v.ref = allele_string;
        }
    }

    // shift our variant back if it's an indel
    if (!substitution) {
        --v.position;
        assert(v.position >= 1);
    }

    v.updateAlleleIndexes();

    return trav_to_allele;
}

void Deconstructor::get_genotypes(vcflib::Variant& v, const vector<string>& names,
                                  const vector<int>& trav_to_allele) {
    map<string, int> name_to_allele;
    for (int i = 0; i < names.size(); ++i) {
        name_to_allele[names[i]] = trav_to_allele[i];
    }
    v.format.push_back("GT");
    for (auto& sample_name : sample_names) {
        auto na = name_to_allele.find(sample_name);
        if (na != name_to_allele.end()) {
            v.samples[sample_name]["GT"] = {std::to_string(na->second)};
        } else {
            v.samples[sample_name]["GT"] = {"."};
        }
    }
}

bool Deconstructor::deconstruct_site(const Snarl* snarl) {

    auto contents = snarl_manager->shallow_contents(snarl, *graph, false);
    if (contents.first.empty()) {
        // Nothing but the boundary nodes in this snarl
#ifdef debug
#pragma omp critical (cerr)
        cerr << "Skipping empty site " << pb2json(*snarl) << endl;
#endif
        return false;
    }

    // find every traversal that runs through a path in the graph
    pair<vector<SnarlTraversal>, vector<string>> named_travs;
    named_travs = path_trav_finder->find_named_traversals(*snarl);

    // pick out the traversal corresponding to a reference path, breaking ties consistently
    int ref_trav_idx = -1;
    for (int i = 0; i < named_travs.first.size(); ++i) {
        if (pindexes.count(named_travs.second.at(i)) &&
            (ref_trav_idx == -1 || named_travs.second[i] < named_travs.second[ref_trav_idx])) {
            ref_trav_idx = i;
        }
    }

    // there's no reference path through the snarl, so we can't make a variant
    // (todo: should we try to detect this before computing traversals?)
    if (ref_trav_idx == -1) {
#ifdef debug
#pragma omp critical (cerr)
        cerr << "Skipping site becuase no reference traversal was found " << pb2json(*snarl) << endl;
#endif
        return false;
    }

    // add in the exhaustive traversals
    if (!path_restricted) {
        // exhaustive traversal can't do all snarls
        if (!snarl->type() == ULTRABUBBLE) {
            return false;
        }
        if (!check_max_nodes(snarl)) {
#pragma omp critical (cerr)
            cerr << "Warning: Skipping site because it is too complex for exhaustive traversal enumeration: " << pb2json(*snarl) << endl << "         Consider using -e to traverse embedded paths" << endl;
            return false;
        }
        vector<SnarlTraversal> exhaustive_travs = explicit_exhaustive_traversals(snarl);
        // happens when there was a nested non-ultrabubble snarl
        if (exhaustive_travs.empty()) {
            return false;
        }
        named_travs.first.insert(named_travs.first.end(), exhaustive_travs.begin(), exhaustive_travs.end());
        for (int i = 0; i < exhaustive_travs.size(); ++i) {
            // dummy names so we can use the same code as the named path traversals above
            named_travs.second.push_back(" >>" + std::to_string(i));
        }
    }

    // there's not alt path through the snarl, so we can't make an interesting variant
    if (named_travs.first.size() < 2) {
#ifdef debug
#pragma omp critical (cerr)
        cerr << "Skipping site because to alt traversal was found " << pb2json(*snarl) << endl;
#endif
        return false;
    }

    vcflib::Variant v;
    v.setVariantCallFile(outvcf);
    v.quality = 23;

    // write variant's sequenceName (VCF contig)
    v.sequenceName = named_travs.second.at(ref_trav_idx);

    // Set position based on the lowest position in the snarl.
    pair<size_t, bool> pos_orientation_start = pindexes[v.sequenceName]->by_id[snarl->start().node_id()];
    pair<size_t, bool> pos_orientation_end = pindexes[v.sequenceName]->by_id[snarl->end().node_id()];
    bool use_start = pos_orientation_start.first < pos_orientation_end.first;
    size_t node_pos = (use_start ? pos_orientation_start.first : pos_orientation_end.first);
    v.position = node_pos + 1; // shift from 0-based to 1-based
    if (use_start) {
        v.position += graph->get_node(snarl->start().node_id())->sequence().length();
    } else {
        v.position += graph->get_node(snarl->end().node_id())->sequence().length();
    }

    // We need to shift back a position for indels.  Get the previous base from the graph:
    const Visit& prev_visit = use_start ? snarl->start() : snarl->end();
    int offset = 0;
    if (use_start != prev_visit.backward()) {
        offset = graph->get_length(graph->get_handle(prev_visit.node_id())) - 1;
    }
    char prev_char = ::toupper(graph->get_sequence(graph->get_handle(prev_visit.node_id()))[offset]);

    // Convert the snarl traversals to strings and add them to the variant
    vector<int> trav_to_allele = get_alleles(v, named_travs.first, ref_trav_idx, prev_char);

    // Fill in the genotypes
    if (path_restricted) {
        get_genotypes(v, named_travs.second, trav_to_allele);
    }

#pragma omp critical (cout)
    {
        cout << v << endl;
    }

    return true;
}

/**
 * Convenience wrapper function for deconstruction of multiple paths.
 */
void Deconstructor::deconstruct(vector<string> ref_paths, vg::VG* graph, SnarlManager* snarl_manager,
                                bool path_restricted_traversals) {

    this->graph = graph;
    this->snarl_manager = snarl_manager;
    this->path_restricted = path_restricted_traversals;

    // Create path index for the contig if we don't have one.
    for (auto& refpath : ref_paths) {
        if (pindexes.find(refpath) == pindexes.end()){
            pindexes[refpath] = new PathIndex(*graph, refpath, false);
        }
    }

    // Keep track of the non-reference paths in the graph.  They'll be our sample names
    sample_names.clear();
    graph->for_each_path_handle([&](const path_handle_t& path_handle) {
            string path_name = graph->get_path_name(path_handle);
            if (!pindexes.count(path_name)) {
                sample_names.push_back(path_name);
            }
        });

    // print the VCF header
    stringstream stream;
    stream << "##fileformat=VCFv4.2" << endl;
    if (path_restricted) {
        stream << "##FORMAT=<ID=GT,Number=1,Type=String,Description=\"Genotype\">" << endl;
    }
    for(auto& refpath : ref_paths) {
        size_t path_len = 0;
        path_handle_t path_handle = graph->get_path_handle(refpath);
        for (handle_t handle : graph->scan_path(path_handle)) {
            path_len += graph->get_length(handle);
        }
        stream << "##contig=<ID=" << refpath << ",length=" << path_len << ">" << endl;
    }
    stream << "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT";
    if (path_restricted) {
        for (auto& sample_name : sample_names) {
            stream << "\t" << sample_name;
        }
    }
    stream << endl;

    string hstr = stream.str();
    assert(outvcf.openForOutput(hstr));
    cout << outvcf.header << endl;

    // create the traversal finder
    map<string, const Alignment*> reads_by_name;
    path_trav_finder = unique_ptr<PathRestrictedTraversalFinder>(new PathRestrictedTraversalFinder(*graph,
                                                                                *snarl_manager,
                                                                                reads_by_name,
                                                                                1,
                                                                                100,
                                                                                true));

    if (!path_restricted) {
        trav_finder = unique_ptr<TraversalFinder>(new ExhaustiveTraversalFinder(*graph,
                                                                                *snarl_manager,
                                                                                true));

    }

    // Do the top-level snarls in parallel
    snarl_manager->for_each_top_level_snarl_parallel([&](const Snarl* snarl) {
            vector<const Snarl*> todo(1, snarl);
            vector<const Snarl*> next;
            while (!todo.empty()) {
                for (auto next_snarl : todo) {
                    // if we can't make a variant from the snarl due to not finding
                    // paths through it, we try again on the children
                    // note: we may want to push the parallelism down a bit
                    if (!deconstruct_site(next_snarl)) {
                        const vector<const Snarl*>& children = snarl_manager->children_of(next_snarl);
                        next.insert(next.end(), children.begin(), children.end());
                    }
                }
                swap(todo, next);
                next.clear();
            }
        });
}

bool Deconstructor::check_max_nodes(const Snarl* snarl)  {
    unordered_set<Node*> nodeset = snarl_manager->deep_contents(snarl, *graph, false).first;
    int node_count = 0;
    for (auto node : nodeset) {
        if (graph->start_degree(node) > 1 || graph->end_degree(node) > 1) {
            ++node_count;
            if (node_count > max_nodes_for_exhaustive) {
                return false;
            }
        }
    }
    return true;
};

vector<SnarlTraversal> Deconstructor::explicit_exhaustive_traversals(const Snarl* snarl){
    vector<SnarlTraversal> out_travs;
    bool ultra_all_the_way_down = true;
    function<void(const SnarlTraversal&, const Snarl&)> extend_trav =
        [&](const SnarlTraversal& trav, const Snarl& nested_snarl) {
        // exhaustive traversal finder is limited.  if we find something
        // that's not an ultrabubble, not much we can do
        if (nested_snarl.type() != ULTRABUBBLE) {
            ultra_all_the_way_down = false;
            return;
        }
        vector<SnarlTraversal> nested_travs = trav_finder->find_traversals(nested_snarl);
        for (auto& nested_trav : nested_travs) {
            SnarlTraversal extended_trav = trav;
            bool is_explicit = true;
            for (int i = 0; i < nested_trav.visit_size(); ++i) {
                if (nested_trav.visit(i).node_id() != 0) {
                    Visit* visit = extended_trav.add_visit();
                    *visit = nested_trav.visit(i);
                } else {
                    extend_trav(extended_trav, nested_trav.visit(i).snarl());
                    is_explicit = false;
                }
            }
            if (is_explicit) {
                out_travs.push_back(extended_trav);
            }
        }
    };
    SnarlTraversal trav;
    extend_trav(trav, *snarl);
    if (!ultra_all_the_way_down) {
        out_travs.clear();
    }
    return out_travs;
}

}

<|endoftext|>