<commit_before>// Simple startrails composition program using OpenCV
// Copyright 2018 Jarno Paananen <jarno.paananen@gmail.com>
// Based on script by Thomas Jacquin
// SPDX-License-Identifier: MIT

using namespace std;

#include <getopt.h>
#include <glob.h>
#include <sys/resource.h>
#include <sys/time.h>
#include <unistd.h>
#include <algorithm>
#include <cstdlib>
#include <iostream>
#include <mutex>
#include <string>
#include <thread>
#include <vector>

#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/opencv.hpp>

#define KNRM "\x1B[0m"
#define KRED "\x1B[31m"
#define KGRN "\x1B[32m"
#define KYEL "\x1B[33m"
#define KBLU "\x1B[34m"
#define KMAG "\x1B[35m"
#define KCYN "\x1B[36m"
#define KWHT "\x1B[37m"

struct config_t {
  std::string img_src_dir, img_src_ext, dst_startrails;
  bool startrails_enabled;
  int num_threads;
  int nice_level;
  int img_width;
  int img_height;
  int verbose;
  double brightness_limit;
} config;

std::mutex stdio_mutex;

void parse_args(int, char**, struct config_t*);
void usage_and_exit(int);
void startrail_worker(int,               // thread num
                      struct config_t*,  // config
                      glob_t*,           // file list
                      std::mutex*,       // mutex
                      cv::Mat*,          // statistics
                      cv::Mat*           // accumulated
);

void startrail_worker(int thread_num,
                      struct config_t* cf,
                      glob_t* files,
                      std::mutex* mtx,
                      cv::Mat* stats_ptr,
                      cv::Mat* main_accumulator) {
  int start_num, end_num, batch_size, nchan = 0;
  unsigned long nfiles = files->gl_pathc;
  cv::Mat thread_accumulator;

  batch_size = nfiles / cf->num_threads;
  start_num = thread_num * batch_size;

  // last thread has more work to do if the number of images isn't multiple of
  // the number of threads
  if ((thread_num + 1) == cf->num_threads)
    end_num = nfiles - 1;
  else
    end_num = start_num + batch_size - 1;

  if (cf->verbose > 1) {
    stdio_mutex.lock();
    fprintf(stderr, "thread %d/%d processing files %d-%d (%d/%lu)\n",
            thread_num + 1, cf->num_threads, start_num, end_num,
            end_num - start_num + 1, nfiles);
    stdio_mutex.unlock();
  }

  for (int f = start_num; f <= end_num; f++) {
    char* filename = files->gl_pathv[f];
    cv::Mat image = cv::imread(filename, cv::IMREAD_UNCHANGED);
    filename = basename(filename);
    if (!image.data) {
      stdio_mutex.lock();
      fprintf(stderr, "Error reading file %s\n", filename);
      stdio_mutex.unlock();
      continue;
    }

    if (cf->img_height && cf->img_width &&
        (image.cols != cf->img_width || image.rows != cf->img_height)) {
      if (cf->verbose) {
        stdio_mutex.lock();
        fprintf(stderr, "skip %s size %dx%d != %dx%d\n", filename, image.cols,
                image.cols, cf->img_width, cf->img_height);
        stdio_mutex.unlock();
      }
      continue;
    }

    // first valid image sets the number of channels we expect
    if (nchan == 0 && image.channels())
      nchan = image.channels();

    cv::Scalar mean_scalar = cv::mean(image);
    double image_mean;
    switch (image.channels()) {
      default:  // mono case
        image_mean = mean_scalar.val[0];
        break;
      case 3:  // for color choose maximum channel
      case 4:
        image_mean =
            cv::max(mean_scalar[0], cv::max(mean_scalar[1], mean_scalar[2]));
        break;
    }
    // Scale to 0-1 range
    switch (image.depth()) {
      case CV_8U:
        image_mean /= 255.0;
        break;
      case CV_16U:
        image_mean /= 65535.0;
        break;
    }
    if (cf->verbose > 1) {
      stdio_mutex.lock();
      fprintf(stderr, "[%d/%lu] %s %.3f\n", f + 1, nfiles, filename,
              image_mean);
      stdio_mutex.unlock();
    }

    // the matrix pointed to by stats_ptr has already been initialized to NAN
    // so we just update the entry once the image is successfully loaded
    stats_ptr->col(f) = image_mean;

    if (cf->startrails_enabled && image_mean <= cf->brightness_limit) {
      if (image.channels() != nchan) {
        if (cf->verbose) {
          stdio_mutex.lock();
          fprintf(stderr, "repairing channel mismatch: %d != %d\n",
                  image.channels(), nchan);
          stdio_mutex.unlock();
        }
        if (image.channels() < nchan)
          cv::cvtColor(image, image, cv::COLOR_GRAY2BGR, nchan);
        else if (image.channels() > nchan)
          cv::cvtColor(image, image, cv::COLOR_BGR2GRAY, nchan);
      }
      if (thread_accumulator.empty()) {
        image.copyTo(thread_accumulator);
      } else {
        thread_accumulator = cv::max(thread_accumulator, image);
      }
    }
  }

  if (cf->startrails_enabled) {
    // skip unlucky threads that might have got only bad images
    if (!thread_accumulator.empty()) {
      mtx->lock();
      if (main_accumulator->empty()) {
        thread_accumulator.copyTo(*main_accumulator);
      } else {
        *main_accumulator = cv::max(thread_accumulator, *main_accumulator);
      }
      mtx->unlock();
    }
  }
}

void parse_args(int argc, char** argv, struct config_t* cf) {
  int c, tmp;
  cf->verbose = 0;
  cf->startrails_enabled = true;
  cf->img_height = cf->img_width = 0;
  cf->brightness_limit = 0.35;  // not terrible in the city
  cf->nice_level = 10;
  cf->num_threads = std::thread::hardware_concurrency();

  while (1) {  // getopt loop
    int option_index = 0;
    static struct option long_options[] = {
        {"brightness", required_argument, 0, 'b'},
        {"directory", required_argument, 0, 'd'},
        {"extension", required_argument, 0, 'e'},
        {"max-threads", required_argument, 0, 'm'},
        {"nice-level", required_argument, 0, 'n'},
        {"output", required_argument, 0, 'o'},
        {"image-size", required_argument, 0, 's'},
        {"statistics", no_argument, 0, 'S'},
        {"verbose", no_argument, 0, 'v'},
        {"help", no_argument, 0, 'h'},
        {0, 0, 0, 0}};

    c = getopt_long(argc, argv, "hvSb:d:e:m:n:o:s:", long_options,
                    &option_index);
    if (c == -1)
      break;
    switch (c) {  // option switch
      case 'h':
        usage_and_exit(0);
        // NOTREACHED
        break;
      case 'v':
        cf->verbose++;
        break;
      case 'S':
        cf->startrails_enabled = false;
        break;
      case 's':
        int height, width;
        sscanf(optarg, "%dx%d", &width, &height);
        // 122.8Mpx should be enough for anybody.
        if (height < 0 || height > 9600 || width < 0 || width > 12800)
          height = width = 0;
        cf->img_height = height;
        cf->img_width = width;
        break;
      case 'b':
        double b;
        b = atof(optarg);
        if (b >= 0 && b <= 1.0)
          cf->brightness_limit = b;
        break;
      case 'd':
        cf->img_src_dir = optarg;
        break;
      case 'e':
        cf->img_src_ext = optarg;
        break;
      case 'm':
        tmp = atoi(optarg);
        if ((tmp >= 1) && (tmp < cf->num_threads))
          cf->num_threads = tmp;
        else
          fprintf(stderr, "invalid number of threads %d; using %d\n", tmp,
                  cf->num_threads);
        break;
      case 'n':
        tmp = atoi(optarg);
        if (PRIO_MIN > tmp) {
          tmp = PRIO_MIN;
          fprintf(stderr, "clamping scheduler priority to PRIO_MIN\n");
        } else if (PRIO_MAX < tmp) {
          fprintf(stderr, "clamping scheduler priority to PRIO_MAX\n");
          tmp = PRIO_MAX;
        }
        cf->nice_level = atoi(optarg);
        break;
      case 'o':
        cf->dst_startrails = optarg;
        break;
      default:
        break;
    }  // option switch
  }    // getopt loop
}

void usage_and_exit(int x) {
  std::cout << "Usage: startrails [-v] -d <dir> -e <ext> [-b <brightness> -o "
               "<output> | -s] [-m <max-threads>] [-n <nice>]"
            << std::endl;
  if (x) {
    std::cout << KRED
              << "Source directory and file extension are always required."
              << std::endl;
    std::cout << "brightness threshold and output file are required to render "
                 "startrails"
              << KNRM << std::endl;
  }

  std::cout << std::endl << "Arguments:" << std::endl;
  std::cout << "-h | --help : display this help, then exit" << std::endl;
  std::cout << "-v | --verbose : increase log verbosity" << std::endl;
  std::cout << "-S | --statistics : print image directory statistics without "
               "producing image."
            << std::endl;
  std::cout << "-d | --directory <str> : directory from which to read images"
            << std::endl;
  std::cout << "-e | --extension <str> : filter images to just this extension"
            << std::endl;
  std::cout << "-m | --max-threads <int> : limit maximum number of processing "
               "threads. (0 = nproc)"
            << std::endl;
  std::cout << "-n | --nice <int> : nice(2) level of processing threads (10)"
            << std::endl;
  std::cout << "-o | --output-file <str> : output image filename" << std::endl;
  std::cout << "-s | --image-size <int>x<int> : restrict processed images to "
               "this size"
            << std::endl;
  std::cout << "-b | --brightness-limit <float> : ranges from 0 (black) to 1 "
               "(white). Default 0.35"
            << std::endl;
  std::cout << "\tA moonless sky may be as low as 0.05 while full moon can be "
               "as high as 0.4"
            << std::endl;
  std::cout << std::endl
            << "ex: startrails -b 0.07 -d ../images/20180208/ -e jpg -o "
               "startrails.jpg"
            << std::endl;
  exit(x);
}

int main(int argc, char* argv[]) {
  int r;
  struct config_t config;
  int i;
  char* e;

  parse_args(argc, argv, &config);

  if (config.verbose < 1)
    if ((e = getenv("ALLSKY_DEBUG_LEVEL")))
      if ((i = atoi(e)) > 0)
        config.verbose = i;

  if (config.img_src_dir.empty() || config.img_src_ext.empty())
    usage_and_exit(3);

  if (!config.startrails_enabled) {
    config.brightness_limit = 0;
    config.dst_startrails = "/dev/null";
  }

  if (!config.dst_startrails.empty() && config.brightness_limit < 0)
    usage_and_exit(3);

  r = setpriority(PRIO_PROCESS, 0, config.nice_level);
  if (r) {
    config.nice_level = getpriority(PRIO_PROCESS, 0);
    fprintf(stderr, "unable to set nice level: %s\n", strerror(errno));
  }

  // Find files
  glob_t files;
  std::string wildcard = config.img_src_dir + "/*." + config.img_src_ext;
  glob(wildcard.c_str(), 0, NULL, &files);
  if (files.gl_pathc == 0) {
    globfree(&files);
    std::cout << "No images found, exiting." << std::endl;
    return 0;
  }

  std::mutex accumulated_mutex;
  cv::Mat accumulated;
  cv::Mat stats;
  stats.create(1, files.gl_pathc, CV_64F);
  // initialize stats to NAN because some images might legitimately be 100%
  // brightness if they're massively overexposed. They should be counted in
  // the summary statistics. It is not entirely accurate to signal invalid
  // images with 1.0 brightness since no image data was read.
  stats = NAN;

  std::vector<std::thread> threadpool;
  for (int tid = 0; tid < config.num_threads; tid++)
    threadpool.push_back(std::thread(startrail_worker, tid, &config, &files,
                                     &accumulated_mutex, &stats, &accumulated));

  for (auto& t : threadpool)
    t.join();

  // Calculate some descriptive statistics
  double ds_min, ds_max, ds_mean, ds_median;
  cv::Point min_loc;

  // Each thread will have updated stats with the brightness of the images
  // that were successfully processed. Invalid images will be left as NAN.
  // In OpenCV, NAN is unequal to everything including NAN which means we can
  // filter out bogus entries by checking stats for element-wise equality
  // with itself.
  cv::Mat nan_mask = cv::Mat(stats == stats);
  cv::Mat filtered_stats;
  stats.copyTo(filtered_stats, nan_mask);
  cv::minMaxLoc(stats, &ds_min, &ds_max, &min_loc);
  ds_mean = cv::mean(filtered_stats)[0];

  // For median, do partial sort and take middle value
  std::vector<double> vec;
  filtered_stats.copyTo(vec);
  std::nth_element(vec.begin(), vec.begin() + (vec.size() / 2), vec.end());
  ds_median = vec[vec.size() / 2];

  std::cout << "Minimum: " << ds_min << " maximum: " << ds_max
            << " mean: " << ds_mean << " median: " << ds_median << std::endl;

  // If we still don't have an image (no images below threshold), copy the
  // minimum mean image so we see why
  if (config.startrails_enabled) {
    if (accumulated.empty()) {
      fprintf(
          stderr,
          "No images below threshold %.3f, writing the minimum image only\n",
          config.brightness_limit);
      accumulated = cv::imread(files.gl_pathv[min_loc.x], cv::IMREAD_UNCHANGED);
    }

    std::vector<int> compression_params;
    compression_params.push_back(cv::IMWRITE_PNG_COMPRESSION);
    compression_params.push_back(9);
    compression_params.push_back(cv::IMWRITE_JPEG_QUALITY);
    compression_params.push_back(95);

    cv::imwrite(config.dst_startrails, accumulated, compression_params);
  }
  globfree(&files);
  return 0;
}
<commit_msg>better concurrency limit check<commit_after>// Simple startrails composition program using OpenCV
// Copyright 2018 Jarno Paananen <jarno.paananen@gmail.com>
// Based on script by Thomas Jacquin
// SPDX-License-Identifier: MIT

using namespace std;

#include <getopt.h>
#include <glob.h>
#include <sys/resource.h>
#include <sys/time.h>
#include <unistd.h>
#include <algorithm>
#include <cstdlib>
#include <iostream>
#include <mutex>
#include <string>
#include <thread>
#include <vector>

#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/opencv.hpp>

#define KNRM "\x1B[0m"
#define KRED "\x1B[31m"
#define KGRN "\x1B[32m"
#define KYEL "\x1B[33m"
#define KBLU "\x1B[34m"
#define KMAG "\x1B[35m"
#define KCYN "\x1B[36m"
#define KWHT "\x1B[37m"

struct config_t {
  std::string img_src_dir, img_src_ext, dst_startrails;
  bool startrails_enabled;
  int num_threads;
  int nice_level;
  int img_width;
  int img_height;
  int verbose;
  double brightness_limit;
} config;

std::mutex stdio_mutex;

void parse_args(int, char**, struct config_t*);
void usage_and_exit(int);
void startrail_worker(int,               // thread num
                      struct config_t*,  // config
                      glob_t*,           // file list
                      std::mutex*,       // mutex
                      cv::Mat*,          // statistics
                      cv::Mat*           // accumulated
);

void startrail_worker(int thread_num,
                      struct config_t* cf,
                      glob_t* files,
                      std::mutex* mtx,
                      cv::Mat* stats_ptr,
                      cv::Mat* main_accumulator) {
  int start_num, end_num, batch_size, nchan = 0;
  unsigned long nfiles = files->gl_pathc;
  cv::Mat thread_accumulator;

  batch_size = nfiles / cf->num_threads;
  start_num = thread_num * batch_size;

  // last thread has more work to do if the number of images isn't multiple of
  // the number of threads
  if ((thread_num + 1) == cf->num_threads)
    end_num = nfiles - 1;
  else
    end_num = start_num + batch_size - 1;

  if (cf->verbose > 1) {
    stdio_mutex.lock();
    fprintf(stderr, "thread %d/%d processing files %d-%d (%d/%lu)\n",
            thread_num + 1, cf->num_threads, start_num, end_num,
            end_num - start_num + 1, nfiles);
    stdio_mutex.unlock();
  }

  for (int f = start_num; f <= end_num; f++) {
    char* filename = files->gl_pathv[f];
    cv::Mat image = cv::imread(filename, cv::IMREAD_UNCHANGED);
    filename = basename(filename);
    if (!image.data) {
      stdio_mutex.lock();
      fprintf(stderr, "Error reading file %s\n", filename);
      stdio_mutex.unlock();
      continue;
    }

    if (cf->img_height && cf->img_width &&
        (image.cols != cf->img_width || image.rows != cf->img_height)) {
      if (cf->verbose) {
        stdio_mutex.lock();
        fprintf(stderr, "skip %s size %dx%d != %dx%d\n", filename, image.cols,
                image.cols, cf->img_width, cf->img_height);
        stdio_mutex.unlock();
      }
      continue;
    }

    // first valid image sets the number of channels we expect
    if (nchan == 0 && image.channels())
      nchan = image.channels();

    cv::Scalar mean_scalar = cv::mean(image);
    double image_mean;
    switch (image.channels()) {
      default:  // mono case
        image_mean = mean_scalar.val[0];
        break;
      case 3:  // for color choose maximum channel
      case 4:
        image_mean =
            cv::max(mean_scalar[0], cv::max(mean_scalar[1], mean_scalar[2]));
        break;
    }
    // Scale to 0-1 range
    switch (image.depth()) {
      case CV_8U:
        image_mean /= 255.0;
        break;
      case CV_16U:
        image_mean /= 65535.0;
        break;
    }
    if (cf->verbose > 1) {
      stdio_mutex.lock();
      fprintf(stderr, "[%d/%lu] %s %.3f\n", f + 1, nfiles, filename,
              image_mean);
      stdio_mutex.unlock();
    }

    // the matrix pointed to by stats_ptr has already been initialized to NAN
    // so we just update the entry once the image is successfully loaded
    stats_ptr->col(f) = image_mean;

    if (cf->startrails_enabled && image_mean <= cf->brightness_limit) {
      if (image.channels() != nchan) {
        if (cf->verbose) {
          stdio_mutex.lock();
          fprintf(stderr, "repairing channel mismatch: %d != %d\n",
                  image.channels(), nchan);
          stdio_mutex.unlock();
        }
        if (image.channels() < nchan)
          cv::cvtColor(image, image, cv::COLOR_GRAY2BGR, nchan);
        else if (image.channels() > nchan)
          cv::cvtColor(image, image, cv::COLOR_BGR2GRAY, nchan);
      }
      if (thread_accumulator.empty()) {
        image.copyTo(thread_accumulator);
      } else {
        thread_accumulator = cv::max(thread_accumulator, image);
      }
    }
  }

  if (cf->startrails_enabled) {
    // skip unlucky threads that might have got only bad images
    if (!thread_accumulator.empty()) {
      mtx->lock();
      if (main_accumulator->empty()) {
        thread_accumulator.copyTo(*main_accumulator);
      } else {
        *main_accumulator = cv::max(thread_accumulator, *main_accumulator);
      }
      mtx->unlock();
    }
  }
}

void parse_args(int argc, char** argv, struct config_t* cf) {
  int c, tmp, ncpu = std::thread::hardware_concurrency();
  cf->verbose = 0;
  cf->startrails_enabled = true;
  cf->img_height = cf->img_width = 0;
  cf->brightness_limit = 0.35;  // not terrible in the city
  cf->nice_level = 10;
  cf->num_threads = ncpu;

  while (1) {  // getopt loop
    int option_index = 0;
    static struct option long_options[] = {
        {"brightness", required_argument, 0, 'b'},
        {"directory", required_argument, 0, 'd'},
        {"extension", required_argument, 0, 'e'},
        {"max-threads", required_argument, 0, 'm'},
        {"nice-level", required_argument, 0, 'n'},
        {"output", required_argument, 0, 'o'},
        {"image-size", required_argument, 0, 's'},
        {"statistics", no_argument, 0, 'S'},
        {"verbose", no_argument, 0, 'v'},
        {"help", no_argument, 0, 'h'},
        {0, 0, 0, 0}};

    c = getopt_long(argc, argv, "hvSb:d:e:m:n:o:s:", long_options,
                    &option_index);
    if (c == -1)
      break;
    switch (c) {  // option switch
      case 'h':
        usage_and_exit(0);
        // NOTREACHED
        break;
      case 'v':
        cf->verbose++;
        break;
      case 'S':
        cf->startrails_enabled = false;
        break;
      case 's':
        int height, width;
        sscanf(optarg, "%dx%d", &width, &height);
        // 122.8Mpx should be enough for anybody.
        if (height < 0 || height > 9600 || width < 0 || width > 12800)
          height = width = 0;
        cf->img_height = height;
        cf->img_width = width;
        break;
      case 'b':
        double b;
        b = atof(optarg);
        if (b >= 0 && b <= 1.0)
          cf->brightness_limit = b;
        break;
      case 'd':
        cf->img_src_dir = optarg;
        break;
      case 'e':
        cf->img_src_ext = optarg;
        break;
      case 'm':
        tmp = atoi(optarg);
        if ((tmp >= 1) && (tmp <= ncpu))
          cf->num_threads = tmp;
        else
          fprintf(stderr, "invalid number of threads %d; using %d\n", tmp,
                  cf->num_threads);
        break;
      case 'n':
        tmp = atoi(optarg);
        if (PRIO_MIN > tmp) {
          tmp = PRIO_MIN;
          fprintf(stderr, "clamping scheduler priority to PRIO_MIN\n");
        } else if (PRIO_MAX < tmp) {
          fprintf(stderr, "clamping scheduler priority to PRIO_MAX\n");
          tmp = PRIO_MAX;
        }
        cf->nice_level = atoi(optarg);
        break;
      case 'o':
        cf->dst_startrails = optarg;
        break;
      default:
        break;
    }  // option switch
  }    // getopt loop
}

void usage_and_exit(int x) {
  std::cout << "Usage: startrails [-v] -d <dir> -e <ext> [-b <brightness> -o "
               "<output> | -s] [-m <max-threads>] [-n <nice>]"
            << std::endl;
  if (x) {
    std::cout << KRED
              << "Source directory and file extension are always required."
              << std::endl;
    std::cout << "brightness threshold and output file are required to render "
                 "startrails"
              << KNRM << std::endl;
  }

  std::cout << std::endl << "Arguments:" << std::endl;
  std::cout << "-h | --help : display this help, then exit" << std::endl;
  std::cout << "-v | --verbose : increase log verbosity" << std::endl;
  std::cout << "-S | --statistics : print image directory statistics without "
               "producing image."
            << std::endl;
  std::cout << "-d | --directory <str> : directory from which to read images"
            << std::endl;
  std::cout << "-e | --extension <str> : filter images to just this extension"
            << std::endl;
  std::cout << "-m | --max-threads <int> : limit maximum number of processing "
               "threads. (0 = nproc)"
            << std::endl;
  std::cout << "-n | --nice <int> : nice(2) level of processing threads (10)"
            << std::endl;
  std::cout << "-o | --output-file <str> : output image filename" << std::endl;
  std::cout << "-s | --image-size <int>x<int> : restrict processed images to "
               "this size"
            << std::endl;
  std::cout << "-b | --brightness-limit <float> : ranges from 0 (black) to 1 "
               "(white). Default 0.35"
            << std::endl;
  std::cout << "\tA moonless sky may be as low as 0.05 while full moon can be "
               "as high as 0.4"
            << std::endl;
  std::cout << std::endl
            << "ex: startrails -b 0.07 -d ../images/20180208/ -e jpg -o "
               "startrails.jpg"
            << std::endl;
  exit(x);
}

int main(int argc, char* argv[]) {
  int r;
  struct config_t config;
  int i;
  char* e;

  parse_args(argc, argv, &config);

  if (config.verbose < 1)
    if ((e = getenv("ALLSKY_DEBUG_LEVEL")))
      if ((i = atoi(e)) > 0)
        config.verbose = i;

  if (config.img_src_dir.empty() || config.img_src_ext.empty())
    usage_and_exit(3);

  if (!config.startrails_enabled) {
    config.brightness_limit = 0;
    config.dst_startrails = "/dev/null";
  }

  if (!config.dst_startrails.empty() && config.brightness_limit < 0)
    usage_and_exit(3);

  r = setpriority(PRIO_PROCESS, 0, config.nice_level);
  if (r) {
    config.nice_level = getpriority(PRIO_PROCESS, 0);
    fprintf(stderr, "unable to set nice level: %s\n", strerror(errno));
  }

  // Find files
  glob_t files;
  std::string wildcard = config.img_src_dir + "/*." + config.img_src_ext;
  glob(wildcard.c_str(), 0, NULL, &files);
  if (files.gl_pathc == 0) {
    globfree(&files);
    std::cout << "No images found, exiting." << std::endl;
    return 0;
  }

  std::mutex accumulated_mutex;
  cv::Mat accumulated;
  cv::Mat stats;
  stats.create(1, files.gl_pathc, CV_64F);
  // initialize stats to NAN because some images might legitimately be 100%
  // brightness if they're massively overexposed. They should be counted in
  // the summary statistics. It is not entirely accurate to signal invalid
  // images with 1.0 brightness since no image data was read.
  stats = NAN;

  std::vector<std::thread> threadpool;
  for (int tid = 0; tid < config.num_threads; tid++)
    threadpool.push_back(std::thread(startrail_worker, tid, &config, &files,
                                     &accumulated_mutex, &stats, &accumulated));

  for (auto& t : threadpool)
    t.join();

  // Calculate some descriptive statistics
  double ds_min, ds_max, ds_mean, ds_median;
  cv::Point min_loc;

  // Each thread will have updated stats with the brightness of the images
  // that were successfully processed. Invalid images will be left as NAN.
  // In OpenCV, NAN is unequal to everything including NAN which means we can
  // filter out bogus entries by checking stats for element-wise equality
  // with itself.
  cv::Mat nan_mask = cv::Mat(stats == stats);
  cv::Mat filtered_stats;
  stats.copyTo(filtered_stats, nan_mask);
  cv::minMaxLoc(stats, &ds_min, &ds_max, &min_loc);
  ds_mean = cv::mean(filtered_stats)[0];

  // For median, do partial sort and take middle value
  std::vector<double> vec;
  filtered_stats.copyTo(vec);
  std::nth_element(vec.begin(), vec.begin() + (vec.size() / 2), vec.end());
  ds_median = vec[vec.size() / 2];

  std::cout << "Minimum: " << ds_min << " maximum: " << ds_max
            << " mean: " << ds_mean << " median: " << ds_median << std::endl;

  // If we still don't have an image (no images below threshold), copy the
  // minimum mean image so we see why
  if (config.startrails_enabled) {
    if (accumulated.empty()) {
      fprintf(
          stderr,
          "No images below threshold %.3f, writing the minimum image only\n",
          config.brightness_limit);
      accumulated = cv::imread(files.gl_pathv[min_loc.x], cv::IMREAD_UNCHANGED);
    }

    std::vector<int> compression_params;
    compression_params.push_back(cv::IMWRITE_PNG_COMPRESSION);
    compression_params.push_back(9);
    compression_params.push_back(cv::IMWRITE_JPEG_QUALITY);
    compression_params.push_back(95);

    cv::imwrite(config.dst_startrails, accumulated, compression_params);
  }
  globfree(&files);
  return 0;
}
<|endoftext|>


<commit_before>/*
** This file is part of libyuni, a cross-platform C++ framework (http://libyuni.org).
**
** This Source Code Form is subject to the terms of the Mozilla Public License
** v.2.0. If a copy of the MPL was not distributed with this file, You can
** obtain one at http://mozilla.org/MPL/2.0/.
**
** github: https://github.com/libyuni/libyuni/
** gitlab: https://gitlab.com/libyuni/libyuni/ (mirror)
*/
#include "service.h"
#include "../../core/system/cpu.h"
#include "../../thread/array.h"
#include "../../private/jobs/queue/thread.h"
#ifndef YUNI_OS_WINDOWS
	#include <unistd.h>
#endif

namespace Yuni
{
namespace Job
{

	namespace
	{

	constexpr static uint32_t maxNumberOfThreads = 512;

	using ThreadArray = Yuni::Thread::Array<Yuni::Private::QueueService::QueueThread>;

	inline uint32_t optimalCPUCount()
	{
		auto count = System::CPU::Count();
		return (count > 2) ? (count - 1) : (count < 1 ? 1 : count);
	}

	} // anonymous namespace

	QueueService::QueueService()
	{
		auto count = optimalCPUCount();
		pMinimumThreadCount = count;
		pMaximumThreadCount = count;
	}

	QueueService::QueueService(bool autostart)
	{
		auto count = optimalCPUCount();
		pMinimumThreadCount = count;
		pMaximumThreadCount = count;
		if (autostart)
			start();
	}

	QueueService::~QueueService()
	{
		// making sure that the queueservice is stopped before being destroyed
		stop();
	}

	bool QueueService::maximumThreadCount(uint count)
	{
		if (count > maxNumberOfThreads) // hard-coded value
			return false;
		if (0 == count) // default value
			count = optimalCPUCount();

		MutexLocker locker(*this);
		// checking for the lower bound
		if (pMinimumThreadCount > count)
			pMinimumThreadCount = count;
		// reseting the upper bound
		pMaximumThreadCount = count;
		return true;
	}

	uint QueueService::maximumThreadCount() const
	{
		MutexLocker locker(*this);
		return pMaximumThreadCount;
	}

	bool QueueService::minimumThreadCount(uint count)
	{
		if (count > maxNumberOfThreads) // hard-coded value
			return false;
		if (0 == count)
			count = optimalCPUCount();

		MutexLocker locker(*this);

		// checking for the upper bound
		if (pMaximumThreadCount < count)
			pMaximumThreadCount = count;
		// reseting the lower bound
		pMinimumThreadCount = count;
		return true;
	}

	uint QueueService::minimumThreadCount() const
	{
		MutexLocker locker(*this);
		return pMinimumThreadCount;
	}

	std::pair<uint,uint> QueueService::minmaxThreadCount() const
	{
		MutexLocker locker(*this);
		return std::pair<uint, uint>(pMinimumThreadCount, pMaximumThreadCount);
	}

	bool QueueService::minmaxThreadCount(const std::pair<uint, uint>& values)
	{
		uint maxv = values.second;
		if (maxv > maxNumberOfThreads)
			return false;
		uint minv = values.first;
		if (minv > maxv)
			minv = maxv;
		if (maxv == 0)
		{
			maxv = optimalCPUCount();
			minv = maxv;
		}

		MutexLocker locker(*this);
		pMinimumThreadCount = minv;
		pMaximumThreadCount = maxv;
		return true;
	}

	bool QueueService::start()
	{
		MutexLocker locker(*this);
		if (YUNI_LIKELY(pStatus == State::stopped))
		{
			pSignalAllThreadHaveStopped.reset();
			pSignalShouldStop.reset();

			delete (ThreadArray*) pThreads;
			pThreads = new ThreadArray();

			// alias to the thread pool
			ThreadArray& array = *((ThreadArray*) pThreads);
			// recreate the thread pool
			// adding the minimum number of threads
			array.clear();
			for (uint i = 0; i != pMinimumThreadCount; ++i)
				array += new Yuni::Private::QueueService::QueueThread(*this);

			// Start all threads at once
			array.start();
			// Ok now we have started
			pStatus = State::running;
		}
		return true;
	}

	void QueueService::stop(uint timeout)
	{
		ThreadArray* threads; // the thread pool
		// getting the thread pool
		{
			MutexLocker locker(*this);
			if (pStatus != State::running)
				return;

			threads  = (ThreadArray*) pThreads;
			pThreads = nullptr;
			pStatus  = State::stopping;
		}

		// Destroying the thread pool
		if (YUNI_LIKELY(threads))
		{
			// stopping all threads (**before** deleting them)
			threads->stop(timeout);
			delete threads;
		}
		MutexLocker locker(*this);
		// marking the queue service as stopped, just in case
		// (thread workers should already marked us as 'stopped')
		pStatus = State::stopped;
		// signalling that the queueservice is stopped. This signal
		// will come after pSignalAllThreadHaveStopped in this case
		pSignalShouldStop.notify();
	}

	void QueueService::registerWorker(void* threadself)
	{
		assert(threadself != nullptr);
		MutexLocker locker(*this);
		if (pWorkerSet.count(threadself) == 0)
			pWorkerSet.insert(threadself);
	}

	void QueueService::unregisterWorker(void* threadself)
	{
		assert(threadself != nullptr);
		MutexLocker locker(*this);

		Yuni::Set<void*>::Unordered::iterator it = pWorkerSet.find(threadself);
		if (pWorkerSet.end() != it)
		{
			pWorkerSet.erase(it);
			if (pWorkerSet.empty())
			{
				if (pStatus == State::stopping)
					pStatus = State::stopped;
				pSignalAllThreadHaveStopped.notify();
			}
		}
	}

	bool QueueService::restart(uint timeout)
	{
		stop(timeout);
		return start();
	}

	void QueueService::gracefulStop()
	{
		MutexLocker locker(*this);
		if (pThreads and pStatus == State::running)
		{
			// about to stop
			pStatus = State::stopping;
			// ask to stop to all threads
			((ThreadArray*) pThreads)->gracefulStop();
			// notifying that the queueservice is stopped (or will stop soon)
			pSignalShouldStop.notify();
		}
	}

	inline bool QueueService::waitForAllThreads(uint timeout)
	{
		// waiting for all threads to stop
		do
		{
			if (0 == timeout)
			{
				pSignalAllThreadHaveStopped.wait();
			}
			else
			{
				if (not pSignalAllThreadHaveStopped.wait(timeout))
					return false;
			}

			MutexLocker locker(*this);
			// if the queue is running, we may have to reset our internal state
			if (pStatus == State::running)
			{
				if (not pWorkerSet.empty() or not pWaitingRoom.empty())
				{
					pSignalAllThreadHaveStopped.reset();
					continue;
				}
			}
			break;
		}
		while (true);
		return true;
	}

	void QueueService::wait(QServiceEvent event)
	{
		assert((event == qseIdle or event == qseStop) and "invalid event");

		// checking if not started
		{
			MutexLocker locker(*this);
			if (pStatus == State::stopped)
				return;
		}

		switch (event)
		{
			case qseStop:
			{
				// waiting for being terminated
				pSignalShouldStop.wait();

				// break : do not break - waiting for all threads being stopped
			}
			case qseIdle:
			{
				waitForAllThreads(0);
				break;
			}
			default:
			{
				assert(false and "invalid value for event");
			}
		}
	}

	bool QueueService::wait(QServiceEvent event, uint timeout)
	{
		assert((event == qseIdle or event == qseStop) and "invalid event");
		assert(timeout > 0 and "invalid timeout");

		// checking if not started
		{
			MutexLocker locker(*this);
			if (pStatus == State::stopped)
				return true;
		}

		switch (event)
		{
			case qseStop:
			{
				// waiting for being terminated
				if (not pSignalShouldStop.wait(timeout))
					return false;

				waitForAllThreads(0);
				break;
			}
			case qseIdle:
			{
				if (not waitForAllThreads(timeout))
					return false;
				break;
			}
			default:
			{
				assert(false and "invalid value for event");
			}
		}
		return true;
	}

	inline void QueueService::wakeupWorkers()
	{
		MutexLocker locker(*this);
		if (pWorkerSet.size() < pMaximumThreadCount and pThreads)
			((ThreadArray*) pThreads)->wakeUp();
	}

	void QueueService::add(const IJob::Ptr& job)
	{
		assert(!(!job) and "invalid job");
		pWaitingRoom.add(job);
		wakeupWorkers();
	}

	void QueueService::add(const IJob::Ptr& job, Priority priority)
	{
		assert(!(!job) and "invalid job");
		pWaitingRoom.add(job, priority);
		wakeupWorkers();
	}

	void QueueService::clear()
	{
		pWaitingRoom.clear();
	}

	uint QueueService::threadCount() const
	{
		MutexLocker locker(*this);
		return pThreads ? ((const ThreadArray*) pThreads)->size() : 0;
	}

	namespace // anonymous
	{

		struct QueueActivityPredicate final
		{
			using ThreadInfoType = QueueService::ThreadInfo;
			using VectorType = ThreadInfoType::Vector;

			QueueActivityPredicate(VectorType* out)
				: pList(out)
			{
				pList->clear();
			}

			template<class ThreadPtrT>
			bool operator () (const ThreadPtrT& thread) const
			{
				ThreadInfoType* info = new ThreadInfoType();
				info->thread = thread;
				if (!(!(info->thread)))
				{
					info->job = thread->currentJob();
					if (!(!(info->job)))
					{
						// We have a job which is currently working !
						info->hasJob = true;
						info->job->fillInformation(*info);
						pList->push_back(info);
						return true;
					}
				}
				info->hasJob = false;
				info->state = Yuni::Job::State::idle;
				info->canceling = false;
				info->progression = 0;
				pList->push_back(info);
				return true;
			}

		private:
			mutable VectorType* pList;
		};

	} // anonymous namespace

	void QueueService::activitySnapshot(QueueService::ThreadInfo::Vector& out)
	{
		QueueActivityPredicate predicate(&out);
		MutexLocker locker(*this);
		if (pThreads)
			((ThreadArray*) pThreads)->foreachThread(predicate);
	}

} // namespace Job
} // namespace Yuni
<commit_msg>jobs: unique_ptr for the threads pointer in QueueService::stop<commit_after>/*
** This file is part of libyuni, a cross-platform C++ framework (http://libyuni.org).
**
** This Source Code Form is subject to the terms of the Mozilla Public License
** v.2.0. If a copy of the MPL was not distributed with this file, You can
** obtain one at http://mozilla.org/MPL/2.0/.
**
** github: https://github.com/libyuni/libyuni/
** gitlab: https://gitlab.com/libyuni/libyuni/ (mirror)
*/
#include "service.h"
#include "../../core/system/cpu.h"
#include "../../thread/array.h"
#include "../../private/jobs/queue/thread.h"
#ifndef YUNI_OS_WINDOWS
	#include <unistd.h>
#endif
#include <memory>

namespace Yuni
{
namespace Job
{

	namespace
	{

	constexpr static uint32_t maxNumberOfThreads = 512;

	using ThreadArray = Yuni::Thread::Array<Yuni::Private::QueueService::QueueThread>;

	inline uint32_t optimalCPUCount()
	{
		auto count = System::CPU::Count();
		return (count > 2) ? (count - 1) : (count < 1 ? 1 : count);
	}

	} // anonymous namespace

	QueueService::QueueService()
	{
		auto count = optimalCPUCount();
		pMinimumThreadCount = count;
		pMaximumThreadCount = count;
	}

	QueueService::QueueService(bool autostart)
	{
		auto count = optimalCPUCount();
		pMinimumThreadCount = count;
		pMaximumThreadCount = count;
		if (autostart)
			start();
	}

	QueueService::~QueueService()
	{
		// making sure that the queueservice is stopped before being destroyed
		stop();
	}

	bool QueueService::maximumThreadCount(uint count)
	{
		if (count > maxNumberOfThreads) // hard-coded value
			return false;
		if (0 == count) // default value
			count = optimalCPUCount();

		MutexLocker locker(*this);
		// checking for the lower bound
		if (pMinimumThreadCount > count)
			pMinimumThreadCount = count;
		// reseting the upper bound
		pMaximumThreadCount = count;
		return true;
	}

	uint QueueService::maximumThreadCount() const
	{
		MutexLocker locker(*this);
		return pMaximumThreadCount;
	}

	bool QueueService::minimumThreadCount(uint count)
	{
		if (count > maxNumberOfThreads) // hard-coded value
			return false;
		if (0 == count)
			count = optimalCPUCount();

		MutexLocker locker(*this);

		// checking for the upper bound
		if (pMaximumThreadCount < count)
			pMaximumThreadCount = count;
		// reseting the lower bound
		pMinimumThreadCount = count;
		return true;
	}

	uint QueueService::minimumThreadCount() const
	{
		MutexLocker locker(*this);
		return pMinimumThreadCount;
	}

	std::pair<uint,uint> QueueService::minmaxThreadCount() const
	{
		MutexLocker locker(*this);
		return std::pair<uint, uint>(pMinimumThreadCount, pMaximumThreadCount);
	}

	bool QueueService::minmaxThreadCount(const std::pair<uint, uint>& values)
	{
		uint maxv = values.second;
		if (maxv > maxNumberOfThreads)
			return false;
		uint minv = values.first;
		if (minv > maxv)
			minv = maxv;
		if (maxv == 0)
		{
			maxv = optimalCPUCount();
			minv = maxv;
		}

		MutexLocker locker(*this);
		pMinimumThreadCount = minv;
		pMaximumThreadCount = maxv;
		return true;
	}

	bool QueueService::start()
	{
		MutexLocker locker(*this);
		if (YUNI_LIKELY(pStatus == State::stopped))
		{
			pSignalAllThreadHaveStopped.reset();
			pSignalShouldStop.reset();

			delete (ThreadArray*) pThreads;
			pThreads = new ThreadArray();

			// alias to the thread pool
			ThreadArray& array = *((ThreadArray*) pThreads);
			// recreate the thread pool
			// adding the minimum number of threads
			array.clear();
			for (uint i = 0; i != pMinimumThreadCount; ++i)
				array += new Yuni::Private::QueueService::QueueThread(*this);

			// Start all threads at once
			array.start();
			// Ok now we have started
			pStatus = State::running;
		}
		return true;
	}

	void QueueService::stop(uint timeout)
	{
		std::unique_ptr<ThreadArray> threads; // the thread pool
		// getting the thread pool
		{
			MutexLocker locker(*this);
			if (pStatus != State::running)
				return;
			threads.reset((ThreadArray*) pThreads);
			pThreads = nullptr;
			pStatus  = State::stopping;
		}

		// Destroying the thread pool
		if (YUNI_LIKELY(threads))
		{
			// stopping all threads (**before** deleting them)
			threads->stop(timeout);
			threads.reset(nullptr);
		}
		MutexLocker locker(*this);
		// marking the queue service as stopped, just in case
		// (thread workers should already marked us as 'stopped')
		pStatus = State::stopped;
		// signalling that the queueservice is stopped. This signal
		// will come after pSignalAllThreadHaveStopped in this case
		pSignalShouldStop.notify();
	}

	void QueueService::registerWorker(void* threadself)
	{
		assert(threadself != nullptr);
		MutexLocker locker(*this);
		if (pWorkerSet.count(threadself) == 0)
			pWorkerSet.insert(threadself);
	}

	void QueueService::unregisterWorker(void* threadself)
	{
		assert(threadself != nullptr);
		MutexLocker locker(*this);

		Yuni::Set<void*>::Unordered::iterator it = pWorkerSet.find(threadself);
		if (pWorkerSet.end() != it)
		{
			pWorkerSet.erase(it);
			if (pWorkerSet.empty())
			{
				if (pStatus == State::stopping)
					pStatus = State::stopped;
				pSignalAllThreadHaveStopped.notify();
			}
		}
	}

	bool QueueService::restart(uint timeout)
	{
		stop(timeout);
		return start();
	}

	void QueueService::gracefulStop()
	{
		MutexLocker locker(*this);
		if (pThreads and pStatus == State::running)
		{
			// about to stop
			pStatus = State::stopping;
			// ask to stop to all threads
			((ThreadArray*) pThreads)->gracefulStop();
			// notifying that the queueservice is stopped (or will stop soon)
			pSignalShouldStop.notify();
		}
	}

	inline bool QueueService::waitForAllThreads(uint timeout)
	{
		// waiting for all threads to stop
		do
		{
			if (0 == timeout)
			{
				pSignalAllThreadHaveStopped.wait();
			}
			else
			{
				if (not pSignalAllThreadHaveStopped.wait(timeout))
					return false;
			}

			MutexLocker locker(*this);
			// if the queue is running, we may have to reset our internal state
			if (pStatus == State::running)
			{
				if (not pWorkerSet.empty() or not pWaitingRoom.empty())
				{
					pSignalAllThreadHaveStopped.reset();
					continue;
				}
			}
			break;
		}
		while (true);
		return true;
	}

	void QueueService::wait(QServiceEvent event)
	{
		assert((event == qseIdle or event == qseStop) and "invalid event");

		// checking if not started
		{
			MutexLocker locker(*this);
			if (pStatus == State::stopped)
				return;
		}

		switch (event)
		{
			case qseStop:
			{
				// waiting for being terminated
				pSignalShouldStop.wait();

				// break : do not break - waiting for all threads being stopped
			}
			case qseIdle:
			{
				waitForAllThreads(0);
				break;
			}
			default:
			{
				assert(false and "invalid value for event");
			}
		}
	}

	bool QueueService::wait(QServiceEvent event, uint timeout)
	{
		assert((event == qseIdle or event == qseStop) and "invalid event");
		assert(timeout > 0 and "invalid timeout");

		// checking if not started
		{
			MutexLocker locker(*this);
			if (pStatus == State::stopped)
				return true;
		}

		switch (event)
		{
			case qseStop:
			{
				// waiting for being terminated
				if (not pSignalShouldStop.wait(timeout))
					return false;

				waitForAllThreads(0);
				break;
			}
			case qseIdle:
			{
				if (not waitForAllThreads(timeout))
					return false;
				break;
			}
			default:
			{
				assert(false and "invalid value for event");
			}
		}
		return true;
	}

	inline void QueueService::wakeupWorkers()
	{
		MutexLocker locker(*this);
		if (pWorkerSet.size() < pMaximumThreadCount and pThreads)
			((ThreadArray*) pThreads)->wakeUp();
	}

	void QueueService::add(const IJob::Ptr& job)
	{
		assert(!(!job) and "invalid job");
		pWaitingRoom.add(job);
		wakeupWorkers();
	}

	void QueueService::add(const IJob::Ptr& job, Priority priority)
	{
		assert(!(!job) and "invalid job");
		pWaitingRoom.add(job, priority);
		wakeupWorkers();
	}

	void QueueService::clear()
	{
		pWaitingRoom.clear();
	}

	uint QueueService::threadCount() const
	{
		MutexLocker locker(*this);
		return pThreads ? ((const ThreadArray*) pThreads)->size() : 0;
	}

	namespace // anonymous
	{

		struct QueueActivityPredicate final
		{
			using ThreadInfoType = QueueService::ThreadInfo;
			using VectorType = ThreadInfoType::Vector;

			QueueActivityPredicate(VectorType* out)
				: pList(out)
			{
				pList->clear();
			}

			template<class ThreadPtrT>
			bool operator () (const ThreadPtrT& thread) const
			{
				ThreadInfoType* info = new ThreadInfoType();
				info->thread = thread;
				if (!(!(info->thread)))
				{
					info->job = thread->currentJob();
					if (!(!(info->job)))
					{
						// We have a job which is currently working !
						info->hasJob = true;
						info->job->fillInformation(*info);
						pList->push_back(info);
						return true;
					}
				}
				info->hasJob = false;
				info->state = Yuni::Job::State::idle;
				info->canceling = false;
				info->progression = 0;
				pList->push_back(info);
				return true;
			}

		private:
			mutable VectorType* pList;
		};

	} // anonymous namespace

	void QueueService::activitySnapshot(QueueService::ThreadInfo::Vector& out)
	{
		QueueActivityPredicate predicate(&out);
		MutexLocker locker(*this);
		if (pThreads)
			((ThreadArray*) pThreads)->foreachThread(predicate);
	}

} // namespace Job
} // namespace Yuni
<|endoftext|>


<commit_before>//
// Created by zts on 16-12-5.
//

#include "ssdb_impl.h"

static void eset_internal(const SSDBImpl *ssdb, const Bytes &key, int64_t ts);

static int eset_one(SSDBImpl *ssdb, const Bytes &key, int64_t ts);

int SSDBImpl::eset(const Bytes &key, int64_t ts) {
    Transaction trans(binlogs);

    int ret = check_meta_key(key);
    if (ret <= 0){
        return ret;
    }

    ret = eset_one(this, key, ts);
    if (ret >= 0) {
        leveldb::Status s = binlogs->commit();
        if (!s.ok()) {
            log_error("eset error: %s", s.ToString().c_str());
            return -1;
        }
    }

    return ret;
}

int SSDBImpl::esetNoLock(const Bytes &key, int64_t ts) {
    int ret = eset_one(this, key, ts);
    if (ret >= 0) {
        leveldb::Status s = binlogs->commit();
        if (!s.ok()) {
            log_error("eset error: %s", s.ToString().c_str());
            return -1;
        }
    }

    return ret;
}

int SSDBImpl::edel(const Bytes &key) {
    RecordLock l(&mutex_record_, key.String());
    leveldb::WriteBatch batch;

    int ret = edel_one(batch, key);
    if (ret >= 0) {
        leveldb::Status s = ldb->Write(leveldb::WriteOptions(), &(batch));
        if (!s.ok()) {
            log_error("edel error: %s", s.ToString().c_str());
            return -1;
        }
    }

    return ret;
}

int SSDBImpl::edel_one(leveldb::WriteBatch &batch, const Bytes &key) {

    int64_t old_ts = 0;

    int found = eget(key, &old_ts);
    if (found == -1) {
        return -1;
    } else if (found == 0) {
        return 0;
    } else {
        std::string old_score_key = encode_escore_key(key, static_cast<uint64_t>(old_ts));
        std::string old_eset_key = encode_eset_key(key);

        batch.Delete(old_score_key);
        batch.Delete(old_eset_key);
    }

    return 1;
}

int SSDBImpl::eget(const Bytes &key, int64_t *ts) {
    *ts = 0;

    std::string str_score;
    std::string dbkey = encode_eset_key(key);
    leveldb::Status s = ldb->Get(leveldb::ReadOptions(), dbkey, &str_score);
    if (s.IsNotFound()) {
        return 0;
    }
    if (!s.ok()) {
        log_error("%s", s.ToString().c_str());
        return -1;
    }

    *ts = *((uint64_t *) (str_score.data())); // int64_t -> uint64_t
    return 1;
}

void eset_internal(const SSDBImpl *ssdb, const Bytes &key, int64_t ts) {
    string ekey = encode_eset_key(key);

    string buf;
    buf.append((char *) (&ts), sizeof(int64_t));

    string score_key = encode_escore_key(key, static_cast<uint64_t>(ts));

    ssdb->binlogs->Put(ekey, buf);
    ssdb->binlogs->Put(score_key, "");
    ssdb->binlogs->add_log(BinlogCommand::ESET, ekey);
}

int eset_one(SSDBImpl *ssdb, const Bytes &key, int64_t ts) {

    int ret = 1;

    int64_t old_ts = 0;

    int found = ssdb->eget(key, &old_ts);
    if (found == -1) {
        return -1;
    }

    if (found == 0) {
        eset_internal(ssdb, key, ts);

    } else if (found == 1) {
        if (old_ts == ts) {
            //same
        } else {
            string old_score_key = encode_escore_key(key, static_cast<uint64_t>(old_ts));
            ssdb->binlogs->Delete(old_score_key);
            eset_internal(ssdb, key, ts);
        }
    } else {
        //error
        return -1;
    }

    return ret;

}

int SSDBImpl::check_meta_key(const Bytes &key) {
    std::string meta_key = encode_meta_key(key);
    std::string meta_val;
    leveldb::Status s = ldb->Get(leveldb::ReadOptions(), meta_key, &meta_val);
    if (s.IsNotFound()){
        return 0;
    } else if (!s.ok()){
        return -1;
    } else{
        if (meta_val.size() >= 4 ){
            if (meta_val[3] == KEY_DELETE_MASK){
                return 0;
            }
        } else{
            return -1;
        }
    }
    return 1;
}
<commit_msg>移除eset中得binlog<commit_after>//
// Created by zts on 16-12-5.
//

#include "ssdb_impl.h"

static void eset_internal(const SSDBImpl *ssdb, leveldb::WriteBatch &batch, const Bytes &key, int64_t ts);

static int eset_one(SSDBImpl *ssdb, leveldb::WriteBatch &batch, const Bytes &key, int64_t ts);

int SSDBImpl::eset(const Bytes &key, int64_t ts) {
    RecordLock l(&mutex_record_, key.String());
    return esetNoLock(key, ts);
}

int SSDBImpl::esetNoLock(const Bytes &key, int64_t ts) {

    leveldb::WriteBatch batch;

    int ret = check_meta_key(key);
    if (ret <= 0) {
        return ret;
    }

    ret = eset_one(this, batch, key, ts);
    if (ret >= 0) {
        leveldb::Status s = ldb->Write(leveldb::WriteOptions(), &(batch));
        if (!s.ok()) {
            log_error("eset error: %s", s.ToString().c_str());
            return -1;
        }
    }

    return ret;
}

int SSDBImpl::edel(const Bytes &key) {
    RecordLock l(&mutex_record_, key.String());
    leveldb::WriteBatch batch;

    int ret = edel_one(batch, key);
    if (ret >= 0) {
        leveldb::Status s = ldb->Write(leveldb::WriteOptions(), &(batch));
        if (!s.ok()) {
            log_error("edel error: %s", s.ToString().c_str());
            return -1;
        }
    }

    return ret;
}

int SSDBImpl::edel_one(leveldb::WriteBatch &batch, const Bytes &key) {

    int64_t old_ts = 0;

    int found = eget(key, &old_ts);
    if (found == -1) {
        return -1;
    } else if (found == 0) {
        return 0;
    } else {
        std::string old_score_key = encode_escore_key(key, static_cast<uint64_t>(old_ts));
        std::string old_eset_key = encode_eset_key(key);

        batch.Delete(old_score_key);
        batch.Delete(old_eset_key);
    }

    return 1;
}

int SSDBImpl::eget(const Bytes &key, int64_t *ts) {
    *ts = 0;

    std::string str_score;
    std::string dbkey = encode_eset_key(key);
    leveldb::Status s = ldb->Get(leveldb::ReadOptions(), dbkey, &str_score);
    if (s.IsNotFound()) {
        return 0;
    }
    if (!s.ok()) {
        log_error("%s", s.ToString().c_str());
        return -1;
    }

    *ts = *((uint64_t *) (str_score.data())); // int64_t -> uint64_t
    return 1;
}

void eset_internal(const SSDBImpl *ssdb, leveldb::WriteBatch &batch, const Bytes &key, int64_t ts) {
    string ekey = encode_eset_key(key);

    string buf;
    buf.append((char *) (&ts), sizeof(int64_t));

    string score_key = encode_escore_key(key, static_cast<uint64_t>(ts));

    batch.Put(ekey, buf);
    batch.Put(score_key, "");
}

int eset_one(SSDBImpl *ssdb, leveldb::WriteBatch &batch, const Bytes &key, int64_t ts) {

    int ret = 1;

    int64_t old_ts = 0;

    int found = ssdb->eget(key, &old_ts);
    if (found == -1) {
        return -1;
    }

    if (found == 0) {
        eset_internal(ssdb, batch, key, ts);

    } else if (found == 1) {
        if (old_ts == ts) {
            //same
        } else {
            string old_score_key = encode_escore_key(key, static_cast<uint64_t>(old_ts));
            batch.Delete(old_score_key);
            eset_internal(ssdb, batch, key, ts);
        }
    } else {
        //error
        return -1;
    }

    return ret;

}

int SSDBImpl::check_meta_key(const Bytes &key) {
    std::string meta_key = encode_meta_key(key);
    std::string meta_val;
    leveldb::Status s = ldb->Get(leveldb::ReadOptions(), meta_key, &meta_val);
    if (s.IsNotFound()) {
        return 0;
    } else if (!s.ok()) {
        return -1;
    } else {
        if (meta_val.size() >= 4) {
            if (meta_val[3] == KEY_DELETE_MASK) {
                return 0;
            }
        } else {
            return -1;
        }
    }
    return 1;
}
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Copyright (C) 2015 ScyllaDB
 *
 * Modified by ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include "types.hh"
#include "utils/extremum_tracking.hh"
#include "utils/murmur_hash.hh"
#include "hyperloglog.hh"
#include "db/commitlog/replay_position.hh"
#include <algorithm>

namespace sstables {

static constexpr int TOMBSTONE_HISTOGRAM_BIN_SIZE = 100;

/**
 * ColumnStats holds information about the columns for one partition inside sstable
 */
struct column_stats {
    /** how many atomic cells are there in the partition (every cell in a collection counts)*/
    uint64_t cells_count;
    /** how many columns are there in the partition */
    uint64_t column_count;
    /** how many rows are there in the partition */
    uint64_t rows_count;

    uint64_t start_offset;
    uint64_t partition_size;

    /** the largest/smallest (client-supplied) timestamp in the partition */
    min_max_tracker<api::timestamp_type> timestamp_tracker;
    min_max_tracker<int32_t> local_deletion_time_tracker;
    min_max_tracker<int32_t> ttl_tracker;
    /** histogram of tombstone drop time */
    utils::streaming_histogram tombstone_histogram;

    bool has_legacy_counter_shards;
    bool capped_local_deletion_time = false;

    column_stats() :
        cells_count(0),
        column_count(0),
        rows_count(0),
        start_offset(0),
        partition_size(0),
        timestamp_tracker(),
        local_deletion_time_tracker(std::numeric_limits<int32_t>::max(), std::numeric_limits<int32_t>::max()),
        ttl_tracker(0, 0),
        tombstone_histogram(TOMBSTONE_HISTOGRAM_BIN_SIZE),
        has_legacy_counter_shards(false)
        {
    }

    void reset() {
        *this = column_stats();
    }

    void update_timestamp(api::timestamp_type value) {
        timestamp_tracker.update(value);
    }

    void update_local_deletion_time(int32_t value) {
        local_deletion_time_tracker.update(value);
    }
    void update_local_deletion_time(gc_clock::time_point value) {
        bool capped;
        int32_t ldt = adjusted_local_deletion_time(value, capped);
        update_local_deletion_time(ldt);
        capped_local_deletion_time |= capped;
    }
    void update_local_deletion_time_and_tombstone_histogram(int32_t value) {
        local_deletion_time_tracker.update(value);
        tombstone_histogram.update(value);
    }
    void update_local_deletion_time_and_tombstone_histogram(gc_clock::time_point value) {
        bool capped;
        int32_t ldt = adjusted_local_deletion_time(value, capped);
        update_local_deletion_time_and_tombstone_histogram(ldt);
        capped_local_deletion_time |= capped;
    }
    void update_ttl(int32_t value) {
        ttl_tracker.update(value);
    }
    void update_ttl(gc_clock::duration value) {
        ttl_tracker.update(gc_clock::as_int32(value));
    }
    void update(const deletion_time& dt) {
        assert(!dt.live());
        update_timestamp(dt.marked_for_delete_at);
        update_local_deletion_time_and_tombstone_histogram(dt.local_deletion_time);
    }
    void do_update(const tombstone& t) {
        update_timestamp(t.timestamp);
        update_local_deletion_time_and_tombstone_histogram(t.deletion_time);
    }
    void update(const tombstone& t) {
        if (t) {
            do_update(t);
        }
    }
};

class metadata_collector {
public:
    static constexpr double NO_COMPRESSION_RATIO = -1.0;

    static hll::HyperLogLog hyperloglog(int p, int sp) {
        // FIXME: hll::HyperLogLog doesn't support sparse format, so ignoring parameters by the time being.
        return hll::HyperLogLog();
    }
private:
    // EH of 150 can track a max value of 1697806495183, i.e., > 1.5PB
    utils::estimated_histogram _estimated_partition_size{150};
    // EH of 114 can track a max value of 2395318855, i.e., > 2B cells
    utils::estimated_histogram _estimated_cells_count{114};
    db::replay_position _replay_position;
    min_max_tracker<api::timestamp_type> _timestamp_tracker;
    uint64_t _repaired_at = 0;
    min_max_tracker<int32_t> _local_deletion_time_tracker;
    min_max_tracker<int32_t> _ttl_tracker;
    double _compression_ratio = NO_COMPRESSION_RATIO;
    std::set<int> _ancestors;
    utils::streaming_histogram _estimated_tombstone_drop_time{TOMBSTONE_HISTOGRAM_BIN_SIZE};
    int _sstable_level = 0;
    std::vector<bytes_opt> _min_column_names;
    std::vector<bytes_opt> _max_column_names;
    bool _has_legacy_counter_shards = false;
    uint64_t _columns_count = 0;
    uint64_t _rows_count = 0;

    /**
     * Default cardinality estimation method is to use HyperLogLog++.
     * Parameter here(p=13, sp=25) should give reasonable estimation
     * while lowering bytes required to hold information.
     * See CASSANDRA-5906 for detail.
     */
    hll::HyperLogLog _cardinality = hyperloglog(13, 25);
private:
    /*
     * Convert a vector of bytes into a disk array of disk_string<uint16_t>.
     */
    static void convert(disk_array<uint32_t, disk_string<uint16_t>>&to, std::vector<bytes_opt>&& from) {
        for (auto i = 0U; i < from.size(); i++) {
            if (!from[i]) {
                break;
            }
            disk_string<uint16_t> s;
            s.value = std::move(from[i].value());
            to.elements.push_back(std::move(s));
        }
    }
public:
    void add_key(bytes_view key) {
        long hashed = utils::murmur_hash::hash2_64(key, 0);
        _cardinality.offer_hashed(hashed);
    }

    void add_partition_size(uint64_t partition_size) {
        _estimated_partition_size.add(partition_size);
    }

    void add_cells_count(uint64_t cells_count) {
        _estimated_cells_count.add(cells_count);
    }

    void merge_tombstone_histogram(utils::streaming_histogram& histogram) {
        _estimated_tombstone_drop_time.merge(histogram);
    }

    /**
     * Ratio is compressed/uncompressed and it is
     * if you have 1.x then compression isn't helping
     */
    void add_compression_ratio(uint64_t compressed, uint64_t uncompressed) {
        _compression_ratio = (double) compressed/uncompressed;
    }

    void set_replay_position(const db::replay_position & rp) {
        _replay_position = rp;
    }

    void set_repaired_at(uint64_t repaired_at) {
        _repaired_at = repaired_at;
    }

    void add_ancestor(int generation) {
        _ancestors.insert(generation);
    }

    std::set<int> ancestors() const {
        return _ancestors;
    }

    void sstable_level(int sstable_level) {
        _sstable_level = sstable_level;
    }

    std::vector<bytes_opt>& min_column_names() {
        return _min_column_names;
    }

    std::vector<bytes_opt>& max_column_names() {
        return _max_column_names;
    }

    void update_has_legacy_counter_shards(bool has_legacy_counter_shards) {
        _has_legacy_counter_shards = _has_legacy_counter_shards || has_legacy_counter_shards;
    }

    void update(column_stats&& stats) {
        _timestamp_tracker.update(stats.timestamp_tracker);
        _local_deletion_time_tracker.update(stats.local_deletion_time_tracker);
        _ttl_tracker.update(stats.ttl_tracker);
        add_partition_size(stats.partition_size);
        add_cells_count(stats.cells_count);
        merge_tombstone_histogram(stats.tombstone_histogram);
        update_has_legacy_counter_shards(stats.has_legacy_counter_shards);
        _columns_count += stats.column_count;
        _rows_count += stats.rows_count;
    }

    void construct_compaction(compaction_metadata& m) {
        if (!_ancestors.empty()) {
            m.ancestors.elements = utils::chunked_vector<uint32_t>(_ancestors.begin(), _ancestors.end());
        }
        auto cardinality = _cardinality.get_bytes();
        m.cardinality.elements = utils::chunked_vector<uint8_t>(cardinality.get(), cardinality.get() + cardinality.size());
    }

    void construct_stats(stats_metadata& m) {
        m.estimated_partition_size = std::move(_estimated_partition_size);
        m.estimated_cells_count = std::move(_estimated_cells_count);
        m.position = _replay_position;
        m.min_timestamp = _timestamp_tracker.min();
        m.max_timestamp = _timestamp_tracker.max();
        m.min_local_deletion_time = _local_deletion_time_tracker.min();
        m.max_local_deletion_time = _local_deletion_time_tracker.max();
        m.min_ttl = _ttl_tracker.min();
        m.max_ttl = _ttl_tracker.max();
        m.compression_ratio = _compression_ratio;
        m.estimated_tombstone_drop_time = std::move(_estimated_tombstone_drop_time);
        m.sstable_level = _sstable_level;
        m.repaired_at = _repaired_at;
        convert(m.min_column_names, std::move(_min_column_names));
        convert(m.max_column_names, std::move(_max_column_names));
        m.has_legacy_counter_shards = _has_legacy_counter_shards;
        m.columns_count = _columns_count;
        m.rows_count = _rows_count;
    }
};

}

<commit_msg>sstables/metadata_collector: move the default values to the global tracker<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Copyright (C) 2015 ScyllaDB
 *
 * Modified by ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include "types.hh"
#include "utils/extremum_tracking.hh"
#include "utils/murmur_hash.hh"
#include "hyperloglog.hh"
#include "db/commitlog/replay_position.hh"
#include <algorithm>

namespace sstables {

static constexpr int TOMBSTONE_HISTOGRAM_BIN_SIZE = 100;

/**
 * ColumnStats holds information about the columns for one partition inside sstable
 */
struct column_stats {
    /** how many atomic cells are there in the partition (every cell in a collection counts)*/
    uint64_t cells_count;
    /** how many columns are there in the partition */
    uint64_t column_count;
    /** how many rows are there in the partition */
    uint64_t rows_count;

    uint64_t start_offset;
    uint64_t partition_size;

    /** the largest/smallest (client-supplied) timestamp in the partition */
    min_max_tracker<api::timestamp_type> timestamp_tracker;
    min_max_tracker<int32_t> local_deletion_time_tracker;
    min_max_tracker<int32_t> ttl_tracker;
    /** histogram of tombstone drop time */
    utils::streaming_histogram tombstone_histogram;

    bool has_legacy_counter_shards;
    bool capped_local_deletion_time = false;

    column_stats() :
        cells_count(0),
        column_count(0),
        rows_count(0),
        start_offset(0),
        partition_size(0),
        tombstone_histogram(TOMBSTONE_HISTOGRAM_BIN_SIZE),
        has_legacy_counter_shards(false)
        {
    }

    void reset() {
        *this = column_stats();
    }

    void update_timestamp(api::timestamp_type value) {
        timestamp_tracker.update(value);
    }

    void update_local_deletion_time(int32_t value) {
        local_deletion_time_tracker.update(value);
    }
    void update_local_deletion_time(gc_clock::time_point value) {
        bool capped;
        int32_t ldt = adjusted_local_deletion_time(value, capped);
        update_local_deletion_time(ldt);
        capped_local_deletion_time |= capped;
    }
    void update_local_deletion_time_and_tombstone_histogram(int32_t value) {
        local_deletion_time_tracker.update(value);
        tombstone_histogram.update(value);
    }
    void update_local_deletion_time_and_tombstone_histogram(gc_clock::time_point value) {
        bool capped;
        int32_t ldt = adjusted_local_deletion_time(value, capped);
        update_local_deletion_time_and_tombstone_histogram(ldt);
        capped_local_deletion_time |= capped;
    }
    void update_ttl(int32_t value) {
        ttl_tracker.update(value);
    }
    void update_ttl(gc_clock::duration value) {
        ttl_tracker.update(gc_clock::as_int32(value));
    }
    void update(const deletion_time& dt) {
        assert(!dt.live());
        update_timestamp(dt.marked_for_delete_at);
        update_local_deletion_time_and_tombstone_histogram(dt.local_deletion_time);
    }
    void do_update(const tombstone& t) {
        update_timestamp(t.timestamp);
        update_local_deletion_time_and_tombstone_histogram(t.deletion_time);
    }
    void update(const tombstone& t) {
        if (t) {
            do_update(t);
        }
    }
};

class metadata_collector {
public:
    static constexpr double NO_COMPRESSION_RATIO = -1.0;

    static hll::HyperLogLog hyperloglog(int p, int sp) {
        // FIXME: hll::HyperLogLog doesn't support sparse format, so ignoring parameters by the time being.
        return hll::HyperLogLog();
    }
private:
    // EH of 150 can track a max value of 1697806495183, i.e., > 1.5PB
    utils::estimated_histogram _estimated_partition_size{150};
    // EH of 114 can track a max value of 2395318855, i.e., > 2B cells
    utils::estimated_histogram _estimated_cells_count{114};
    db::replay_position _replay_position;
    min_max_tracker<api::timestamp_type> _timestamp_tracker;
    uint64_t _repaired_at = 0;
    min_max_tracker<int32_t> _local_deletion_time_tracker{std::numeric_limits<int32_t>::max(), std::numeric_limits<int32_t>::max()};
    min_max_tracker<int32_t> _ttl_tracker{0, 0};
    double _compression_ratio = NO_COMPRESSION_RATIO;
    std::set<int> _ancestors;
    utils::streaming_histogram _estimated_tombstone_drop_time{TOMBSTONE_HISTOGRAM_BIN_SIZE};
    int _sstable_level = 0;
    std::vector<bytes_opt> _min_column_names;
    std::vector<bytes_opt> _max_column_names;
    bool _has_legacy_counter_shards = false;
    uint64_t _columns_count = 0;
    uint64_t _rows_count = 0;

    /**
     * Default cardinality estimation method is to use HyperLogLog++.
     * Parameter here(p=13, sp=25) should give reasonable estimation
     * while lowering bytes required to hold information.
     * See CASSANDRA-5906 for detail.
     */
    hll::HyperLogLog _cardinality = hyperloglog(13, 25);
private:
    /*
     * Convert a vector of bytes into a disk array of disk_string<uint16_t>.
     */
    static void convert(disk_array<uint32_t, disk_string<uint16_t>>&to, std::vector<bytes_opt>&& from) {
        for (auto i = 0U; i < from.size(); i++) {
            if (!from[i]) {
                break;
            }
            disk_string<uint16_t> s;
            s.value = std::move(from[i].value());
            to.elements.push_back(std::move(s));
        }
    }
public:
    void add_key(bytes_view key) {
        long hashed = utils::murmur_hash::hash2_64(key, 0);
        _cardinality.offer_hashed(hashed);
    }

    void add_partition_size(uint64_t partition_size) {
        _estimated_partition_size.add(partition_size);
    }

    void add_cells_count(uint64_t cells_count) {
        _estimated_cells_count.add(cells_count);
    }

    void merge_tombstone_histogram(utils::streaming_histogram& histogram) {
        _estimated_tombstone_drop_time.merge(histogram);
    }

    /**
     * Ratio is compressed/uncompressed and it is
     * if you have 1.x then compression isn't helping
     */
    void add_compression_ratio(uint64_t compressed, uint64_t uncompressed) {
        _compression_ratio = (double) compressed/uncompressed;
    }

    void set_replay_position(const db::replay_position & rp) {
        _replay_position = rp;
    }

    void set_repaired_at(uint64_t repaired_at) {
        _repaired_at = repaired_at;
    }

    void add_ancestor(int generation) {
        _ancestors.insert(generation);
    }

    std::set<int> ancestors() const {
        return _ancestors;
    }

    void sstable_level(int sstable_level) {
        _sstable_level = sstable_level;
    }

    std::vector<bytes_opt>& min_column_names() {
        return _min_column_names;
    }

    std::vector<bytes_opt>& max_column_names() {
        return _max_column_names;
    }

    void update_has_legacy_counter_shards(bool has_legacy_counter_shards) {
        _has_legacy_counter_shards = _has_legacy_counter_shards || has_legacy_counter_shards;
    }

    void update(column_stats&& stats) {
        _timestamp_tracker.update(stats.timestamp_tracker);
        _local_deletion_time_tracker.update(stats.local_deletion_time_tracker);
        _ttl_tracker.update(stats.ttl_tracker);
        add_partition_size(stats.partition_size);
        add_cells_count(stats.cells_count);
        merge_tombstone_histogram(stats.tombstone_histogram);
        update_has_legacy_counter_shards(stats.has_legacy_counter_shards);
        _columns_count += stats.column_count;
        _rows_count += stats.rows_count;
    }

    void construct_compaction(compaction_metadata& m) {
        if (!_ancestors.empty()) {
            m.ancestors.elements = utils::chunked_vector<uint32_t>(_ancestors.begin(), _ancestors.end());
        }
        auto cardinality = _cardinality.get_bytes();
        m.cardinality.elements = utils::chunked_vector<uint8_t>(cardinality.get(), cardinality.get() + cardinality.size());
    }

    void construct_stats(stats_metadata& m) {
        m.estimated_partition_size = std::move(_estimated_partition_size);
        m.estimated_cells_count = std::move(_estimated_cells_count);
        m.position = _replay_position;
        m.min_timestamp = _timestamp_tracker.min();
        m.max_timestamp = _timestamp_tracker.max();
        m.min_local_deletion_time = _local_deletion_time_tracker.min();
        m.max_local_deletion_time = _local_deletion_time_tracker.max();
        m.min_ttl = _ttl_tracker.min();
        m.max_ttl = _ttl_tracker.max();
        m.compression_ratio = _compression_ratio;
        m.estimated_tombstone_drop_time = std::move(_estimated_tombstone_drop_time);
        m.sstable_level = _sstable_level;
        m.repaired_at = _repaired_at;
        convert(m.min_column_names, std::move(_min_column_names));
        convert(m.max_column_names, std::move(_max_column_names));
        m.has_legacy_counter_shards = _has_legacy_counter_shards;
        m.columns_count = _columns_count;
        m.rows_count = _rows_count;
    }
};

}
<|endoftext|>


<commit_before>/*
This file is part of the ORK library.
Full copyright and license terms can be found in the LICENSE.txt file.
*/
#pragma once
#include "ork/ork.hpp"

#if ORK_USE_GLM
#    include "glm/fwd.hpp"
#endif // ORK_USE_GLM

namespace ork {

/*
From color.hpp
*/
enum class color_space;
#if ORK_USE_GLM
using color4 = glm::vec4;
#endif // ORK_USE_GLM

/*
From command_line.hpp
*/
#if ORK_USE_BOOST
class command_handler;
#endif // ORK_USE_BOOST

/*
From distribution.hpp
*/
#if ORK_USE_BOOST
class command_handler;
class random;
#endif // ORK_USE_BOOST
template<typename T>
class triangle_distribution;
template<typename T>
class trapezoid_distribution;

/*
From file_utils.hpp
*/
template<class functor, class iter_t, class sort>
struct directory_executer;
template<class functor, class search_type, class sort_type>
struct iterate_directory;
template<class T>
struct directory_range;

/*
From filter.hpp
*/
#if ORK_USE_BOOST
template<unsigned C>
struct ring;
template<unsigned order, unsigned inverse_alpha>
struct butterworth;
#endif // ORK_USE_BOOST

/*
From geometry.hpp
*/
enum class angle;
template<angle>
struct circle;

namespace GLM {

class bounding_box;
class interval;
struct segment;
class chain;

namespace MC {
struct view;
struct rotated_view;
} // namespace MC

} // namespace GLM

/*
From glm.hpp
*/
namespace GLM {

struct dunit3;
template<typename T>
struct default_epsilon_factor;

} // namespace GLM

/*
From html.hpp
*/
enum class align;
enum class border;

namespace html {

struct exportable;
struct pair;
struct string;
struct heading;
struct page_break;
struct padding;
struct image;
struct style;
struct image_style;
struct div_style;
struct table_style;
struct line_style;
struct style_set;
struct header;
struct line;
struct label;
struct table_element;
struct table;
struct div;
struct body;
struct document;

} // namespace html

/*
From memory.hpp
*/
template<typename T>
struct default_deleter;
template<typename T>
struct singleton_deleter;
template<class T, class D>
class value_ptr;
template<typename T, typename D>
class shared_ptr;

/*
From orientation.hpp
*/
enum class orientation;

/*
From tagger.hpp
*/
class tagger;
class setup_hierarchy;

/*
From text_io.hpp
*/

namespace json {

#if ORK_USE_JSON

class ORK_ORK_API exportable;
class ORK_ORK_API serializable;

#endif // ORK_USE_JSON

} // namespace json
namespace xml {

#if ORK_USE_PUGI

class ORK_ORK_API exportable;
class ORK_ORK_API serializable;

#endif // ORK_USE_PUGI

} // namespace xml
namespace yaml {

#if ORK_USE_YAML

class ORK_ORK_API exportable;
class ORK_ORK_API serializable;

#endif // ORK_USE_YAML

} // namespace yaml

} // namespace ork
<commit_msg>Added text_serializable and type_traits to fwd<commit_after>/*
This file is part of the ORK library.
Full copyright and license terms can be found in the LICENSE.txt file.
*/
#pragma once
#include "ork/ork.hpp"

#if ORK_USE_GLM
#    include "glm/fwd.hpp"
#endif // ORK_USE_GLM

namespace ork {

/*
From color.hpp
*/
enum class color_space;
#if ORK_USE_GLM
using color4 = glm::vec4;
#endif // ORK_USE_GLM

/*
From command_line.hpp
*/
#if ORK_USE_BOOST
class command_handler;
#endif // ORK_USE_BOOST

/*
From distribution.hpp
*/
#if ORK_USE_BOOST
class command_handler;
class random;
#endif // ORK_USE_BOOST
template<typename T>
class triangle_distribution;
template<typename T>
class trapezoid_distribution;

/*
From file_utils.hpp
*/
template<class functor, class iter_t, class sort>
struct directory_executer;
template<class functor, class search_type, class sort_type>
struct iterate_directory;
template<class T>
struct directory_range;

/*
From filter.hpp
*/
#if ORK_USE_BOOST
template<unsigned C>
struct ring;
template<unsigned order, unsigned inverse_alpha>
struct butterworth;
#endif // ORK_USE_BOOST

/*
From geometry.hpp
*/
enum class angle;
template<angle>
struct circle;

namespace GLM {

class bounding_box;
class interval;
struct segment;
class chain;

namespace MC {
struct view;
struct rotated_view;
} // namespace MC

} // namespace GLM

/*
From glm.hpp
*/
namespace GLM {

struct dunit3;
template<typename T>
struct default_epsilon_factor;

} // namespace GLM

/*
From html.hpp
*/
enum class align;
enum class border;

namespace html {

struct exportable;
struct pair;
struct string;
struct heading;
struct page_break;
struct padding;
struct image;
struct style;
struct image_style;
struct div_style;
struct table_style;
struct line_style;
struct style_set;
struct header;
struct line;
struct label;
struct table_element;
struct table;
struct div;
struct body;
struct document;

} // namespace html

/*
From memory.hpp
*/
template<typename T>
struct default_deleter;
template<typename T>
struct singleton_deleter;
template<class T, class D>
class value_ptr;
template<typename T, typename D>
class shared_ptr;

/*
From orientation.hpp
*/
enum class orientation;

/*
From tagger.hpp
*/
class tagger;
class setup_hierarchy;

/*
From text_io.hpp
*/

namespace json {

#if ORK_USE_JSON

class ORK_ORK_API exportable;
class ORK_ORK_API serializable;

#endif // ORK_USE_JSON

} // namespace json
namespace xml {

#if ORK_USE_PUGI

class ORK_ORK_API exportable;
class ORK_ORK_API serializable;

#endif // ORK_USE_PUGI

} // namespace xml
namespace yaml {

#if ORK_USE_YAML

class ORK_ORK_API exportable;
class ORK_ORK_API serializable;

#endif // ORK_USE_YAML

} // namespace yaml

/*
From text_serializable.hpp
*/
class vector;

/*
From type_traits.hpp
*/
template<typename T>
struct signature_traits;

} // namespace ork
<|endoftext|>


<commit_before>/*
 * File:   dom.hpp
 * Author: ivan
 *
 * Created on October 8, 2015, 3:15 PM
 */

#pragma once

#include <vector>
#include <memory>
#include <ostream>
#include <map>

#include <utki/Unique.hpp>
#include <utki/Void.hpp>

#include <papki/File.hpp>

#include "config.hpp"

namespace svgdom{

enum class EUnit{
	UNKNOWN,
	NUMBER,
	PERCENT,
	EM,
	EX,
	PX,
	CM,
	IN,
	PT,
	PC
};

struct Length{
	real value;
	EUnit unit;

	Length(real value = 0, EUnit unit = EUnit::NUMBER) :
			value(value),
			unit(unit)
	{}

	static Length parse(const std::string& str);
};

struct Element : public utki::Unique{
	std::string id;

	virtual ~Element()noexcept{}

	void attribsToStream(std::ostream& s)const;

	virtual void toStream(std::ostream& s, unsigned indent = 0)const = 0;

	std::string toString()const;
};

struct Container{
	std::vector<std::unique_ptr<Element>> children;

	void childrenToStream(std::ostream& s, unsigned indent)const;
};

struct Transformation{
	enum class EType{
		MATRIX,
		TRANSLATE,
		SCALE,
		ROTATE,
		SKEWX,
		SKEWY
	} type;
	union{
		real a;
		real angle;
	};
	union{
		real b;
		real x;
	};
	union{
		real c;
		real y;
	};
	real d, e, f;
};

struct Transformable{
	std::vector<Transformation> transformations;

	void attribsToStream(std::ostream& s)const;

	static decltype(Transformable::transformations) parse(const std::string& str);
};

enum class EStyleProperty{
	UNKNOWN,
	FONT,
	FONT_FAMILY,
	FONT_SIZE,
	FONT_SIZE_ADJUST,
	FONT_STRETCH,
	FONT_STYLE,
	FONT_VARIANT,
	FONT_WEIGHT,
	DIRECTION,
	LETTER_SPACING,
	TEXT_DECORATION,
	UNICODE_BIDI,
	WORD_SPACING,
	CLIP,
	COLOR,
	CURSOR,
	DISPLAY,
	OVERFLOW,
	VISIBILITY,
	CLIP_PATH,
    CLIP_RULE,
    MASK,
    OPACITY,
    ENABLE_BACKGROUND,
    FILTER,
    FLOOD_COLOR,
    FLOOD_OPACITY,
    LIGHTING_COLOR,
    STOP_COLOR,
    STOP_OPACITY,
    POINTER_EVENTS,
    COLOR_INTERPOLATION,
    COLOR_INTERPOLATION_FILTERS,
    COLOR_PROFILE,
    COLOR_RENDERING,
    FILL,
    FILL_OPACITY,
    FILL_RULE,
    IMAGE_RENDERING,
    MARKER,
    MARKER_END,
    MARKER_MID,
    MARKER_START,
    SHAPE_RENDERING,
    STROKE,
    STROKE_DASHARRAY,
    STROKE_DASHOFFSET,
    STROKE_LINECAP,
    STROKE_LINEJOIN,
    STROKE_MITERLIMIT,
    STROKE_OPACITY,
    STROKE_WIDTH,
    TEXT_RENDERING,
    ALIGNMENT_BASELINE,
    BASELINE_SHIFT,
    DOMINANT_BASELINE,
    GLYPH_ORIENTATION_HORIZONTAL,
    GLYPH_ORIENTATION_VERTICAL,
    KERNING,
    TEXT_ANCHOR,
    WRITING_MODE
};

struct StylePropertyValue{
	bool effective = true;
	std::uint32_t integer;
	real floating;
	Length length;
	std::string str;
	std::unique_ptr<utki::Void> d;

	static StylePropertyValue parsePaint(const std::string& str);

	std::string paintToString()const;
};

struct Styleable{
	std::map<EStyleProperty, StylePropertyValue> styles;

	void attribsToStream(std::ostream& s)const;

	static decltype(styles) parse(const std::string& str);

	static std::string propertyToString(EStyleProperty p);
	static EStyleProperty stringToProperty(std::string str);
};

struct GElement : public Element, public Container, public Transformable, public Styleable{
	void toStream(std::ostream& s, unsigned indent = 0)const override;
};

struct Rectangle{
	Length x, y;
	Length width = Length(100, EUnit::PERCENT);
	Length height = Length(100, EUnit::PERCENT);

	void attribsToStream(std::ostream& s)const;
};

struct SvgElement : public Element, public Container, public Rectangle{
	void toStream(std::ostream& s, unsigned indent = 0)const override;
};

struct PathElement : public Element, public Styleable, public Transformable{
	struct Step{
		enum class EType{
			UNKNOWN,
			CLOSE,
			MOVE_ABS,
			MOVE_REL,
			LINE_ABS,
			LINE_REL,
			HORIZONTAL_LINE_ABS,
			HORIZONTAL_LINE_REL,
			VERTICAL_LINE_ABS,
			VERTICAL_LINE_REL,
			CUBIC_ABS,
			CUBIC_REL,
			CUBIC_SMOOTH_ABS,
			CUBIC_SMOOTH_REL,
			QUADRATIC_SMOOT_ABS,
			QUADRATIC_SMOOT_REL,
			ARC_ABS,
			ARC_REL
		};

		real x, y;

		union{
			real x1;
			real rx;
		};

		union{
			real y1;
			real ry;
		};

		union{
			real x2;
			real xAxisRotation;
		};

		union{
			real y2;
			struct{
				bool largeArc;
				bool sweep;
			} flags;
		};
	};
	std::vector<Step> path;

	void toStream(std::ostream& s, unsigned indent = 0)const override;
};

std::unique_ptr<SvgElement> load(const papki::File& f);

}//~namespace

std::ostream& operator<<(std::ostream& s, const svgdom::Length& l);
<commit_msg>stuff<commit_after>/*
 * File:   dom.hpp
 * Author: ivan
 *
 * Created on October 8, 2015, 3:15 PM
 */

#pragma once

#include <vector>
#include <memory>
#include <ostream>
#include <map>

#include <utki/Unique.hpp>
#include <utki/Void.hpp>

#include <papki/File.hpp>

#include "config.hpp"

namespace svgdom{

enum class EUnit{
	UNKNOWN,
	NUMBER,
	PERCENT,
	EM,
	EX,
	PX,
	CM,
	IN,
	PT,
	PC
};

struct Length{
	real value;
	EUnit unit;

	Length(real value = 0, EUnit unit = EUnit::NUMBER) :
			value(value),
			unit(unit)
	{}

	static Length parse(const std::string& str);
};

struct Element : public utki::Unique{
	std::string id;

	virtual ~Element()noexcept{}

	void attribsToStream(std::ostream& s)const;

	virtual void toStream(std::ostream& s, unsigned indent = 0)const = 0;

	std::string toString()const;
};

struct Container{
	std::vector<std::unique_ptr<Element>> children;

	void childrenToStream(std::ostream& s, unsigned indent)const;
};

struct Transformation{
	enum class EType{
		MATRIX,
		TRANSLATE,
		SCALE,
		ROTATE,
		SKEWX,
		SKEWY
	} type;
	union{
		real a;
		real angle;
	};
	union{
		real b;
		real x;
	};
	union{
		real c;
		real y;
	};
	real d, e, f;
};

struct Transformable{
	std::vector<Transformation> transformations;

	void attribsToStream(std::ostream& s)const;

	static decltype(Transformable::transformations) parse(const std::string& str);
};

enum class EStyleProperty{
	UNKNOWN,
	FONT,
	FONT_FAMILY,
	FONT_SIZE,
	FONT_SIZE_ADJUST,
	FONT_STRETCH,
	FONT_STYLE,
	FONT_VARIANT,
	FONT_WEIGHT,
	DIRECTION,
	LETTER_SPACING,
	TEXT_DECORATION,
	UNICODE_BIDI,
	WORD_SPACING,
	CLIP,
	COLOR,
	CURSOR,
	DISPLAY,
	OVERFLOW,
	VISIBILITY,
	CLIP_PATH,
    CLIP_RULE,
    MASK,
    OPACITY,
    ENABLE_BACKGROUND,
    FILTER,
    FLOOD_COLOR,
    FLOOD_OPACITY,
    LIGHTING_COLOR,
    STOP_COLOR,
    STOP_OPACITY,
    POINTER_EVENTS,
    COLOR_INTERPOLATION,
    COLOR_INTERPOLATION_FILTERS,
    COLOR_PROFILE,
    COLOR_RENDERING,
    FILL,
    FILL_OPACITY,
    FILL_RULE,
    IMAGE_RENDERING,
    MARKER,
    MARKER_END,
    MARKER_MID,
    MARKER_START,
    SHAPE_RENDERING,
    STROKE,
    STROKE_DASHARRAY,
    STROKE_DASHOFFSET,
    STROKE_LINECAP,
    STROKE_LINEJOIN,
    STROKE_MITERLIMIT,
    STROKE_OPACITY,
    STROKE_WIDTH,
    TEXT_RENDERING,
    ALIGNMENT_BASELINE,
    BASELINE_SHIFT,
    DOMINANT_BASELINE,
    GLYPH_ORIENTATION_HORIZONTAL,
    GLYPH_ORIENTATION_VERTICAL,
    KERNING,
    TEXT_ANCHOR,
    WRITING_MODE
};

struct StylePropertyValue{
	bool effective = true;
	std::uint32_t integer;
	real floating;
	Length length;
	std::string str;
	std::unique_ptr<utki::Void> d;

	static StylePropertyValue parsePaint(const std::string& str);

	std::string paintToString()const;
};

struct Styleable{
	std::map<EStyleProperty, StylePropertyValue> styles;

	void attribsToStream(std::ostream& s)const;

	static decltype(styles) parse(const std::string& str);

	static std::string propertyToString(EStyleProperty p);
	static EStyleProperty stringToProperty(std::string str);
};

struct GElement : public Element, public Container, public Transformable, public Styleable{
	void toStream(std::ostream& s, unsigned indent = 0)const override;
};

struct Rectangle{
	Length x, y;
	Length width = Length(100, EUnit::PERCENT);
	Length height = Length(100, EUnit::PERCENT);

	void attribsToStream(std::ostream& s)const;
};

struct SvgElement : public Element, public Container, public Rectangle{
	void toStream(std::ostream& s, unsigned indent = 0)const override;
};

struct PathElement : public Element, public Styleable, public Transformable{
	struct Step{
		enum class EType{
			UNKNOWN,
			CLOSE,
			MOVE_ABS,
			MOVE_REL,
			LINE_ABS,
			LINE_REL,
			HORIZONTAL_LINE_ABS,
			HORIZONTAL_LINE_REL,
			VERTICAL_LINE_ABS,
			VERTICAL_LINE_REL,
			CUBIC_ABS,
			CUBIC_REL,
			CUBIC_SMOOTH_ABS,
			CUBIC_SMOOTH_REL,
			QUADRATIC_SMOOT_ABS,
			QUADRATIC_SMOOT_REL,
			ARC_ABS,
			ARC_REL
		} type;

		real x, y;

		union{
			real x1;
			real rx;
		};

		union{
			real y1;
			real ry;
		};

		union{
			real x2;
			real xAxisRotation;
		};

		union{
			real y2;
			struct{
				bool largeArc;
				bool sweep;
			} flags;
		};
	};
	std::vector<Step> path;

	void toStream(std::ostream& s, unsigned indent = 0)const override;
};

std::unique_ptr<SvgElement> load(const papki::File& f);

}//~namespace

std::ostream& operator<<(std::ostream& s, const svgdom::Length& l);
<|endoftext|>