<commit_before>#ifndef XGBOOST_IO_PAGE_ROW_ITER_INL_HPP_
#define XGBOOST_IO_PAGE_ROW_ITER_INL_HPP_
/*!
 * \file page_row_iter-inl.hpp
 * row iterator based on sparse page
 * \author Tianqi Chen
 */
#include "../data.h"
#include "../utils/iterator.h"
#include "../utils/thread_buffer.h"
#include "./simple_fmatrix-inl.hpp"

namespace xgboost {
namespace io {
/*! \brief page structure that can be used to store a rowbatch */
struct RowBatchPage {
 public:
  RowBatchPage(void)  {
    data_ = new int[kPageSize];
    utils::Assert(data_ != NULL, "fail to allocate row batch page");
    this->Clear();
  }
  ~RowBatchPage(void) {
    if (data_ != NULL) delete [] data_;
  }
  /*!
   * \brief Push one row into page
   *  \param row an instance row
   *  \return false or true to push into
   */
  inline bool PushRow(const RowBatch::Inst &row) {
    const size_t dsize = row.length * sizeof(RowBatch::Entry);
    if (FreeBytes() < dsize+ sizeof(int)) return false;
    row_ptr(Size() + 1) = row_ptr(Size()) + row.length;
    memcpy(data_ptr(row_ptr(Size())) , row.data, dsize);
    ++ data_[0];
    return true;
  }
  /*!
   * \brief get a row batch representation from the page
   * \param p_rptr a temporal space that can be used to provide
   *  ind_ptr storage for RowBatch
   * \return a new RowBatch object
   */
  inline RowBatch GetRowBatch(std::vector<size_t> *p_rptr, size_t base_rowid) {
    RowBatch batch;
    batch.base_rowid = base_rowid;
    batch.data_ptr = this->data_ptr(0);
    batch.size = static_cast<size_t>(this->Size());
    std::vector<size_t> &rptr = *p_rptr;
    rptr.resize(this->Size() + 1);
    for (size_t i = 0; i < rptr.size(); ++i) {
      rptr[i] = static_cast<size_t>(this->row_ptr(i));
    }
    batch.ind_ptr = &rptr[0];
    return batch;
  }
  /*! \brief get i-th row from the batch */
  inline RowBatch::Inst operator[](size_t i) {
    return RowBatch::Inst(data_ptr(0) + row_ptr(i),
                          static_cast<bst_uint>(row_ptr(i+1) - row_ptr(i)));
  }
  /*!
   * \brief clear the page, cleanup the content
   */
  inline void Clear(void) {
    memset(&data_[0], 0, sizeof(int) * kPageSize);
  }
  /*!
   * \brief load one page form instream
   * \return true if loading is successful
   */
  inline bool Load(utils::IStream &fi) {
    return fi.Read(&data_[0], sizeof(int) * kPageSize) != 0;
  }
  /*! \brief save one page into outstream */
  inline void Save(utils::IStream &fo) {
    fo.Write(&data_[0], sizeof(int) * kPageSize);
  }
  /*! \return number of elements */
  inline int Size(void) const {
    return data_[0];
  }
  /*! \brief page size 64 MB */
  static const size_t kPageSize = 64 << 8;

 private:
  /*! \return number of elements */
  inline size_t FreeBytes(void) {
    return (kPageSize - (Size() + 2)) * sizeof(int)
        - row_ptr(Size()) * sizeof(RowBatch::Entry) ;
  }
  /*! \brief equivalent row pointer at i */
  inline int& row_ptr(int i) {
    return data_[kPageSize - i - 1];
  }
  inline RowBatch::Entry* data_ptr(int i) {
    return (RowBatch::Entry*)(&data_[1]) + i;
  }
  // content of data
  int *data_;
};
/*! \brief thread buffer iterator */
class ThreadRowPageIterator: public utils::IIterator<RowBatch> {
 public:
  ThreadRowPageIterator(void) {
    itr.SetParam("buffer_size", "4");
    page_ = NULL;
    base_rowid_ = 0;
    isend_ = false;
  }
  virtual ~ThreadRowPageIterator(void) {
  }
  virtual void Init(void) {
  }
  virtual void BeforeFirst(void) {
    itr.BeforeFirst();
    isend_ = false;
    base_rowid_ = 0;
  }
  virtual bool Next(void) {
    if(!this->LoadNextPage()) return false;
    out_ = page_->GetRowBatch(&tmp_ptr_, base_rowid_);
    base_rowid_ += out_.size;
    return true;
  }
  virtual const RowBatch &Value(void) const{
    return out_;
  }
  /*! \brief load and initialize the iterator with fi */
  inline void Load(const utils::FileStream &fi) {
    itr.get_factory().SetFile(fi);
    itr.Init();
    this->BeforeFirst();
  }
  /*!
   * \brief save a row iterator to output stream, in row iterator format
   */
  inline static void Save(utils::IIterator<RowBatch> *iter,
                          utils::IStream &fo) {
    RowBatchPage page;
    iter->BeforeFirst();
    while (iter->Next()) {
      const RowBatch &batch = iter->Value();
      for (size_t i = 0; i < batch.size; ++i) {
        if (!page.PushRow(batch[i])) {
          page.Save(fo);
          page.Clear();
          utils::Check(page.PushRow(batch[i]), "row is too big");
        }
      }
    }
    if (page.Size() != 0) page.Save(fo);
  }
 private:
  // load in next page
  inline bool LoadNextPage(void) {
    ptop_ = 0;
    bool ret = itr.Next(page_);
    isend_ = !ret;
    return ret;
  }
  // base row id
  size_t base_rowid_;
  // temporal ptr
  std::vector<size_t> tmp_ptr_;
  // output data
  RowBatch out_;
  // whether we reach end of file
  bool isend_;
  // page pointer type
  typedef RowBatchPage* PagePtr;
  // loader factory for page
  struct Factory {
   public:
    size_t file_begin_;
    utils::FileStream fi;
    Factory(void) {}
    inline void SetFile(const utils::FileStream &fi) {
      this->fi = fi;
      file_begin_ = this->fi.Tell();
    }
    inline bool Init(void) {
      return true;
    }
    inline void SetParam(const char *name, const char *val) {}
    inline bool LoadNext(PagePtr &val) {
      return val->Load(fi);
    }
    inline PagePtr Create(void) {
      PagePtr a = new RowBatchPage();
      return a;
    }
    inline void FreeSpace(PagePtr &a) {
      delete a;
    }
    inline void Destroy(void) {
      fi.Close();
    }
    inline void BeforeFirst(void) {
      fi.Seek(file_begin_);
    }
  };

 protected:
  PagePtr page_;
  int ptop_;
  utils::ThreadBuffer<PagePtr,Factory> itr;
};

/*! \brief data matrix using page */
class DMatrixPage : public DataMatrix {
 public:
  DMatrixPage(void) : DataMatrix(kMagic) {
    iter_ = new ThreadRowPageIterator();
    fmat_ = new FMatrixS(iter_);
  }
  // virtual destructor
  virtual ~DMatrixPage(void) {
    delete fmat_;
  }
  virtual IFMatrix *fmat(void) const {
    return fmat_;
  }
  /*! \brief load and initialize the iterator with fi */
  inline void Load(utils::FileStream &fi,
                   bool silent = false,
                   const char *fname = NULL){
    int magic;
    utils::Check(fi.Read(&magic, sizeof(magic)) != 0, "invalid input file format");
    utils::Check(magic == kMagic, "invalid format,magic number mismatch");
    this->info.LoadBinary(fi);
    iter_->Load(fi);
    if (!silent) {
      printf("DMatrixPage: %lux%lu matrix is loaded",
             info.num_row(), info.num_col());
      if (fname != NULL) {
        printf(" from %s\n", fname);
      } else {
        printf("\n");
      }
      if (info.group_ptr.size() != 0) {
        printf("data contains %u groups\n", (unsigned)info.group_ptr.size()-1);
      }
    }
  }
  /*! \brief save a DataMatrix as DMatrixPage*/
  inline static void Save(const char* fname, const DataMatrix &mat, bool silent) {
    utils::FileStream fs(utils::FopenCheck(fname, "wb"));
    int magic = kMagic;
    fs.Write(&magic, sizeof(magic));
    mat.info.SaveBinary(fs);
    ThreadRowPageIterator::Save(mat.fmat()->RowIterator(), fs);
    fs.Close();
    if (!silent) {
      printf("DMatrixPage: %lux%lu is saved to %s\n",
             mat.info.num_row(), mat.info.num_col(), fname);
    }
  }
  /*! \brief the real fmatrix */
  FMatrixS *fmat_;
  /*! \brief row iterator */
  ThreadRowPageIterator *iter_;
  /*! \brief magic number used to identify DMatrix */
  static const int kMagic = 0xffffab02;
};
}  // namespace io
}  // namespace xgboost
#endif  // XGBOOST_IO_PAGE_ROW_ITER_INL_HPP_
<commit_msg>seems page is ok, try add col tmr<commit_after>#ifndef XGBOOST_IO_PAGE_ROW_ITER_INL_HPP_
#define XGBOOST_IO_PAGE_ROW_ITER_INL_HPP_
/*!
 * \file page_row_iter-inl.hpp
 * row iterator based on sparse page
 * \author Tianqi Chen
 */
#include "../data.h"
#include "../utils/iterator.h"
#include "../utils/thread_buffer.h"
#include "./simple_fmatrix-inl.hpp"

namespace xgboost {
namespace io {
/*! \brief page structure that can be used to store a rowbatch */
struct RowBatchPage {
 public:
  RowBatchPage(void)  {
    data_ = new int[kPageSize];
    utils::Assert(data_ != NULL, "fail to allocate row batch page");
    this->Clear();
  }
  ~RowBatchPage(void) {
    if (data_ != NULL) delete [] data_;
  }
  /*!
   * \brief Push one row into page
   *  \param row an instance row
   *  \return false or true to push into
   */
  inline bool PushRow(const RowBatch::Inst &row) {
    const size_t dsize = row.length * sizeof(RowBatch::Entry);
    if (FreeBytes() < dsize+ sizeof(int)) return false;
    row_ptr(Size() + 1) = row_ptr(Size()) + row.length;
    memcpy(data_ptr(row_ptr(Size())) , row.data, dsize);
    ++ data_[0];
    return true;
  }
  /*!
   * \brief get a row batch representation from the page
   * \param p_rptr a temporal space that can be used to provide
   *  ind_ptr storage for RowBatch
   * \return a new RowBatch object
   */
  inline RowBatch GetRowBatch(std::vector<size_t> *p_rptr, size_t base_rowid) {
    RowBatch batch;
    batch.base_rowid = base_rowid;
    batch.data_ptr = this->data_ptr(0);
    batch.size = static_cast<size_t>(this->Size());
    std::vector<size_t> &rptr = *p_rptr;
    rptr.resize(this->Size() + 1);
    for (size_t i = 0; i < rptr.size(); ++i) {
      rptr[i] = static_cast<size_t>(this->row_ptr(i));
    }
    batch.ind_ptr = &rptr[0];
    return batch;
  }
  /*! \brief get i-th row from the batch */
  inline RowBatch::Inst operator[](size_t i) {
    return RowBatch::Inst(data_ptr(0) + row_ptr(i),
                          static_cast<bst_uint>(row_ptr(i+1) - row_ptr(i)));
  }
  /*!
   * \brief clear the page, cleanup the content
   */
  inline void Clear(void) {
    memset(&data_[0], 0, sizeof(int) * kPageSize);
  }
  /*!
   * \brief load one page form instream
   * \return true if loading is successful
   */
  inline bool Load(utils::IStream &fi) {
    return fi.Read(&data_[0], sizeof(int) * kPageSize) != 0;
  }
  /*! \brief save one page into outstream */
  inline void Save(utils::IStream &fo) {
    fo.Write(&data_[0], sizeof(int) * kPageSize);
  }
  /*! \return number of elements */
  inline int Size(void) const {
    return data_[0];
  }
  /*! \brief page size 64 MB */
  static const size_t kPageSize = 64 << 18;

 private:
  /*! \return number of elements */
  inline size_t FreeBytes(void) {
    return (kPageSize - (Size() + 2)) * sizeof(int)
        - row_ptr(Size()) * sizeof(RowBatch::Entry) ;
  }
  /*! \brief equivalent row pointer at i */
  inline int& row_ptr(int i) {
    return data_[kPageSize - i - 1];
  }
  inline RowBatch::Entry* data_ptr(int i) {
    return (RowBatch::Entry*)(&data_[1]) + i;
  }
  // content of data
  int *data_;
};
/*! \brief thread buffer iterator */
class ThreadRowPageIterator: public utils::IIterator<RowBatch> {
 public:
  ThreadRowPageIterator(void) {
    itr.SetParam("buffer_size", "2");
    page_ = NULL;
    base_rowid_ = 0;
    isend_ = false;
  }
  virtual ~ThreadRowPageIterator(void) {
  }
  virtual void Init(void) {
  }
  virtual void BeforeFirst(void) {
    itr.BeforeFirst();
    isend_ = false;
    base_rowid_ = 0;
  }
  virtual bool Next(void) {
    if(!this->LoadNextPage()) return false;
    out_ = page_->GetRowBatch(&tmp_ptr_, base_rowid_);
    base_rowid_ += out_.size;
    return true;
  }
  virtual const RowBatch &Value(void) const{
    return out_;
  }
  /*! \brief load and initialize the iterator with fi */
  inline void Load(const utils::FileStream &fi) {
    itr.get_factory().SetFile(fi);
    itr.Init();
    this->BeforeFirst();
  }
  /*!
   * \brief save a row iterator to output stream, in row iterator format
   */
  inline static void Save(utils::IIterator<RowBatch> *iter,
                          utils::IStream &fo) {
    RowBatchPage page;
    iter->BeforeFirst();
    while (iter->Next()) {
      const RowBatch &batch = iter->Value();
      for (size_t i = 0; i < batch.size; ++i) {
        if (!page.PushRow(batch[i])) {
          page.Save(fo);
          page.Clear();
          utils::Check(page.PushRow(batch[i]), "row is too big");
        }
      }
    }
    if (page.Size() != 0) page.Save(fo);
  }
 private:
  // load in next page
  inline bool LoadNextPage(void) {
    ptop_ = 0;
    bool ret = itr.Next(page_);
    isend_ = !ret;
    return ret;
  }
  // base row id
  size_t base_rowid_;
  // temporal ptr
  std::vector<size_t> tmp_ptr_;
  // output data
  RowBatch out_;
  // whether we reach end of file
  bool isend_;
  // page pointer type
  typedef RowBatchPage* PagePtr;
  // loader factory for page
  struct Factory {
   public:
    size_t file_begin_;
    utils::FileStream fi;
    Factory(void) {}
    inline void SetFile(const utils::FileStream &fi) {
      this->fi = fi;
      file_begin_ = this->fi.Tell();
    }
    inline bool Init(void) {
      return true;
    }
    inline void SetParam(const char *name, const char *val) {}
    inline bool LoadNext(PagePtr &val) {
      return val->Load(fi);
    }
    inline PagePtr Create(void) {
      PagePtr a = new RowBatchPage();
      return a;
    }
    inline void FreeSpace(PagePtr &a) {
      delete a;
    }
    inline void Destroy(void) {
      fi.Close();
    }
    inline void BeforeFirst(void) {
      fi.Seek(file_begin_);
    }
  };

 protected:
  PagePtr page_;
  int ptop_;
  utils::ThreadBuffer<PagePtr,Factory> itr;
};

/*! \brief data matrix using page */
class DMatrixPage : public DataMatrix {
 public:
  DMatrixPage(void) : DataMatrix(kMagic) {
    iter_ = new ThreadRowPageIterator();
    fmat_ = new FMatrixS(iter_);
  }
  // virtual destructor
  virtual ~DMatrixPage(void) {
    delete fmat_;
  }
  virtual IFMatrix *fmat(void) const {
    return fmat_;
  }
  /*! \brief load and initialize the iterator with fi */
  inline void Load(utils::FileStream &fi,
                   bool silent = false,
                   const char *fname = NULL){
    int magic;
    utils::Check(fi.Read(&magic, sizeof(magic)) != 0, "invalid input file format");
    utils::Check(magic == kMagic, "invalid format,magic number mismatch");
    this->info.LoadBinary(fi);
    iter_->Load(fi);
    if (!silent) {
      printf("DMatrixPage: %lux%lu matrix is loaded",
             info.num_row(), info.num_col());
      if (fname != NULL) {
        printf(" from %s\n", fname);
      } else {
        printf("\n");
      }
      if (info.group_ptr.size() != 0) {
        printf("data contains %u groups\n", (unsigned)info.group_ptr.size()-1);
      }
    }
  }
  /*! \brief save a DataMatrix as DMatrixPage*/
  inline static void Save(const char* fname, const DataMatrix &mat, bool silent) {
    utils::FileStream fs(utils::FopenCheck(fname, "wb"));
    int magic = kMagic;
    fs.Write(&magic, sizeof(magic));
    mat.info.SaveBinary(fs);
    ThreadRowPageIterator::Save(mat.fmat()->RowIterator(), fs);
    fs.Close();
    if (!silent) {
      printf("DMatrixPage: %lux%lu is saved to %s\n",
             mat.info.num_row(), mat.info.num_col(), fname);
    }
  }
  /*! \brief the real fmatrix */
  FMatrixS *fmat_;
  /*! \brief row iterator */
  ThreadRowPageIterator *iter_;
  /*! \brief magic number used to identify DMatrix */
  static const int kMagic = 0xffffab02;
};
}  // namespace io
}  // namespace xgboost
#endif  // XGBOOST_IO_PAGE_ROW_ITER_INL_HPP_
<|endoftext|>


<commit_before>/*
 * Concatenate several istreams.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#include "istream_cat.hxx"
#include "istream_oo.hxx"
#include "util/Cast.hxx"

#include <assert.h>
#include <stdarg.h>

struct CatIstream;

struct CatInput {
    CatIstream *cat;
    struct istream *istream;

    /* handler */

    size_t OnData(const void *data, size_t length);
    ssize_t OnDirect(FdType type, int fd, size_t max_length);
    void OnEof();
    void OnError(GError *error);
};

struct CatIstream {
    struct istream output;
    bool reading;
    unsigned current, num;
    CatInput inputs[1];

    CatInput &GetCurrent() {
        return inputs[current];
    }

    const CatInput &GetCurrent() const {
        return inputs[current];
    }

    bool IsCurrent(const CatInput &input) const {
        return &GetCurrent() == &input;
    }

    CatInput &Shift() {
        return inputs[current++];
    }

    bool IsEOF() const {
        return current == num;
    }

    void CloseAllInputs() {
        while (!IsEOF()) {
            auto &input = Shift();
            if (input.istream != nullptr)
                istream_close_handler(input.istream);
        }
    }
};

/*
 * istream handler
 *
 */

inline size_t
CatInput::OnData(const void *data, size_t length)
{
    assert(istream != nullptr);

    if (!cat->IsCurrent(*this))
        return 0;

    return istream_invoke_data(&cat->output, data, length);
}

inline ssize_t
CatInput::OnDirect(FdType type, int fd, size_t max_length)
{
    assert(istream != nullptr);
    assert(cat->IsCurrent(*this));

    return istream_invoke_direct(&cat->output, type, fd, max_length);
}

inline void
CatInput::OnEof()
{
    assert(istream != nullptr);
    istream = nullptr;

    if (cat->IsCurrent(*this)) {
        do {
            cat->Shift();
        } while (!cat->IsEOF() && cat->GetCurrent().istream == nullptr);

        if (cat->IsEOF()) {
            istream_deinit_eof(&cat->output);
        } else if (!cat->reading) {
            /* only call istream_read() if this function was not
               called from istream_cat_read() - in this case,
               istream_cat_read() would provide the loop.  This is
               advantageous because we avoid unnecessary recursing. */
            istream_read(cat->GetCurrent().istream);
        }
    }
}

inline void
CatInput::OnError(GError *error)
{
    assert(istream != nullptr);
    istream = nullptr;

    cat->CloseAllInputs();

    istream_deinit_abort(&cat->output, error);
}

/*
 * istream implementation
 *
 */

static inline CatIstream *
istream_to_cat(struct istream *istream)
{
    return &ContainerCast2(*istream, &CatIstream::output);
}

static off_t
istream_cat_available(struct istream *istream, bool partial)
{
    CatIstream *cat = istream_to_cat(istream);
    off_t available = 0;

    for (auto *input = &cat->GetCurrent(), *end = &cat->inputs[cat->num];
         input < end; ++input) {
        if (input->istream == nullptr)
            continue;

        const off_t a = istream_available(input->istream, partial);
        if (a != (off_t)-1)
            available += a;
        else if (!partial)
            /* if the caller wants the exact number of bytes, and
               one input cannot provide it, we cannot provide it
               either */
            return (off_t)-1;
    }

    return available;
}

static void
istream_cat_read(struct istream *istream)
{
    CatIstream *cat = istream_to_cat(istream);

    pool_ref(cat->output.pool);

    cat->reading = true;

    unsigned prev;
    do {
        while (!cat->IsEOF() && cat->GetCurrent().istream == nullptr)
            ++cat->current;

        if (cat->IsEOF()) {
            istream_deinit_eof(&cat->output);
            break;
        }

        istream_handler_set_direct(cat->GetCurrent().istream,
                                   cat->output.handler_direct);

        prev = cat->current;
        istream_read(cat->GetCurrent().istream);
    } while (!cat->IsEOF() && cat->current != prev);

    cat->reading = false;

    pool_unref(cat->output.pool);
}

static int
istream_cat_as_fd(struct istream *istream)
{
    CatIstream *cat = istream_to_cat(istream);

    /* we can safely forward the as_fd() call to our input if it's the
       last one */

    if (cat->current != cat->num - 1)
        /* not on last input */
        return -1;

    auto &i = cat->GetCurrent();
    int fd = istream_as_fd(i.istream);
    if (fd >= 0)
        istream_deinit(&cat->output);

    return fd;
}

static void
istream_cat_close(struct istream *istream)
{
    CatIstream *cat = istream_to_cat(istream);

    cat->CloseAllInputs();
    istream_deinit(&cat->output);
}

static const struct istream_class istream_cat = {
    .available = istream_cat_available,
    .read = istream_cat_read,
    .as_fd = istream_cat_as_fd,
    .close = istream_cat_close,
};

/*
 * constructor
 *
 */

struct istream *
istream_cat_new(struct pool *pool, ...)
{
    unsigned num = 0;
    va_list ap;
    va_start(ap, pool);
    while (va_arg(ap, struct istream *) != nullptr)
        ++num;
    va_end(ap);

    assert(num > 0);

    CatIstream *cat;
    auto p = p_malloc(pool, sizeof(*cat) + (num - 1) * sizeof(cat->inputs));
    cat = new(p) CatIstream();
    istream_init(&cat->output, &istream_cat, pool);
    cat->reading = false;
    cat->current = 0;
    cat->num = num;

    va_start(ap, pool);
    num = 0;
    struct istream *istream;
    while ((istream = va_arg(ap, struct istream *)) != nullptr) {
        assert(!istream_has_handler(istream));

        CatInput *input = &cat->inputs[num++];
        input->cat = cat;

        istream_assign_handler(&input->istream, istream,
                               &MakeIstreamHandler<CatInput>::handler, input,
                               0);
    }
    va_end(ap);

    return &cat->output;
}
<commit_msg>istream_cat: add constructor<commit_after>/*
 * Concatenate several istreams.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#include "istream_cat.hxx"
#include "istream_oo.hxx"
#include "util/Cast.hxx"

#include <assert.h>
#include <stdarg.h>

struct CatIstream;

struct CatInput {
    CatIstream *cat;
    struct istream *istream;

    /* handler */

    size_t OnData(const void *data, size_t length);
    ssize_t OnDirect(FdType type, int fd, size_t max_length);
    void OnEof();
    void OnError(GError *error);
};

struct CatIstream {
    struct istream output;
    bool reading = false;
    unsigned current = 0;
    const unsigned num;
    CatInput inputs[1];

    CatIstream(struct pool &p, unsigned _num, va_list ap);

    CatInput &GetCurrent() {
        return inputs[current];
    }

    const CatInput &GetCurrent() const {
        return inputs[current];
    }

    bool IsCurrent(const CatInput &input) const {
        return &GetCurrent() == &input;
    }

    CatInput &Shift() {
        return inputs[current++];
    }

    bool IsEOF() const {
        return current == num;
    }

    void CloseAllInputs() {
        while (!IsEOF()) {
            auto &input = Shift();
            if (input.istream != nullptr)
                istream_close_handler(input.istream);
        }
    }
};

/*
 * istream handler
 *
 */

inline size_t
CatInput::OnData(const void *data, size_t length)
{
    assert(istream != nullptr);

    if (!cat->IsCurrent(*this))
        return 0;

    return istream_invoke_data(&cat->output, data, length);
}

inline ssize_t
CatInput::OnDirect(FdType type, int fd, size_t max_length)
{
    assert(istream != nullptr);
    assert(cat->IsCurrent(*this));

    return istream_invoke_direct(&cat->output, type, fd, max_length);
}

inline void
CatInput::OnEof()
{
    assert(istream != nullptr);
    istream = nullptr;

    if (cat->IsCurrent(*this)) {
        do {
            cat->Shift();
        } while (!cat->IsEOF() && cat->GetCurrent().istream == nullptr);

        if (cat->IsEOF()) {
            istream_deinit_eof(&cat->output);
        } else if (!cat->reading) {
            /* only call istream_read() if this function was not
               called from istream_cat_read() - in this case,
               istream_cat_read() would provide the loop.  This is
               advantageous because we avoid unnecessary recursing. */
            istream_read(cat->GetCurrent().istream);
        }
    }
}

inline void
CatInput::OnError(GError *error)
{
    assert(istream != nullptr);
    istream = nullptr;

    cat->CloseAllInputs();

    istream_deinit_abort(&cat->output, error);
}

/*
 * istream implementation
 *
 */

static inline CatIstream *
istream_to_cat(struct istream *istream)
{
    return &ContainerCast2(*istream, &CatIstream::output);
}

static off_t
istream_cat_available(struct istream *istream, bool partial)
{
    CatIstream *cat = istream_to_cat(istream);
    off_t available = 0;

    for (auto *input = &cat->GetCurrent(), *end = &cat->inputs[cat->num];
         input < end; ++input) {
        if (input->istream == nullptr)
            continue;

        const off_t a = istream_available(input->istream, partial);
        if (a != (off_t)-1)
            available += a;
        else if (!partial)
            /* if the caller wants the exact number of bytes, and
               one input cannot provide it, we cannot provide it
               either */
            return (off_t)-1;
    }

    return available;
}

static void
istream_cat_read(struct istream *istream)
{
    CatIstream *cat = istream_to_cat(istream);

    pool_ref(cat->output.pool);

    cat->reading = true;

    unsigned prev;
    do {
        while (!cat->IsEOF() && cat->GetCurrent().istream == nullptr)
            ++cat->current;

        if (cat->IsEOF()) {
            istream_deinit_eof(&cat->output);
            break;
        }

        istream_handler_set_direct(cat->GetCurrent().istream,
                                   cat->output.handler_direct);

        prev = cat->current;
        istream_read(cat->GetCurrent().istream);
    } while (!cat->IsEOF() && cat->current != prev);

    cat->reading = false;

    pool_unref(cat->output.pool);
}

static int
istream_cat_as_fd(struct istream *istream)
{
    CatIstream *cat = istream_to_cat(istream);

    /* we can safely forward the as_fd() call to our input if it's the
       last one */

    if (cat->current != cat->num - 1)
        /* not on last input */
        return -1;

    auto &i = cat->GetCurrent();
    int fd = istream_as_fd(i.istream);
    if (fd >= 0)
        istream_deinit(&cat->output);

    return fd;
}

static void
istream_cat_close(struct istream *istream)
{
    CatIstream *cat = istream_to_cat(istream);

    cat->CloseAllInputs();
    istream_deinit(&cat->output);
}

static const struct istream_class istream_cat = {
    .available = istream_cat_available,
    .read = istream_cat_read,
    .as_fd = istream_cat_as_fd,
    .close = istream_cat_close,
};

/*
 * constructor
 *
 */

inline CatIstream::CatIstream(struct pool &p, unsigned _num, va_list ap)
    :num(_num)
{
    istream_init(&output, &istream_cat, &p);

    unsigned i = 0;
    struct istream *istream;
    while ((istream = va_arg(ap, struct istream *)) != nullptr) {
        assert(!istream_has_handler(istream));

        CatInput *input = &inputs[i++];
        input->cat = this;

        istream_assign_handler(&input->istream, istream,
                               &MakeIstreamHandler<CatInput>::handler, input,
                               0);
    }

    assert(i == num);
}

struct istream *
istream_cat_new(struct pool *pool, ...)
{
    unsigned num = 0;
    va_list ap;
    va_start(ap, pool);
    while (va_arg(ap, struct istream *) != nullptr)
        ++num;
    va_end(ap);

    assert(num > 0);

    CatIstream *cat;
    auto p = p_malloc(pool, sizeof(*cat) + (num - 1) * sizeof(cat->inputs));
    va_start(ap, pool);
    cat = new(p) CatIstream(*pool, num, ap);
    va_end(ap);
    return &cat->output;
}
<|endoftext|>


<commit_before>/*
 * This istream filter reads a 32 bit header size from the stream,
 * reads it into a buffer and invokes a callback with the tail of the
 * stream.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#include "sink_header.hxx"
#include "pool.hxx"
#include "istream_internal.hxx"
#include "async.hxx"
#include "util/Cast.hxx"
#include "util/ByteOrder.hxx"

#include <glib.h>

#include <assert.h>
#include <string.h>
#include <stdint.h>

struct HeaderSink {
    struct istream output;

    enum {
        SIZE, HEADER, CALLBACK, DATA
    } state;

    struct istream *input;

    unsigned char size_buffer[4];

    unsigned char *buffer;
    size_t size, position;

    /**
     * How much data of the input is pending to be consumed?  Only
     * valid while state==CALLBACK.
     */
    size_t pending;

    const struct sink_header_handler *handler;
    void *handler_ctx;

    struct async_operation async_operation;
};

static GQuark
sink_header_quark(void)
{
    return g_quark_from_static_string("sink_header");
}

static size_t
header_invoke_callback(HeaderSink *header, size_t consumed)
{
    assert(header->state == HeaderSink::SIZE ||
           header->state == HeaderSink::HEADER);

    header->async_operation.Finished();

    const ScopePoolRef ref(*header->output.pool TRACE_ARGS);

    /* the base value has been set by sink_header_input_data() */
    header->pending += consumed;

    header->state = HeaderSink::CALLBACK;
    header->handler->done(header->buffer, header->size,
                          &header->output,
                          header->handler_ctx);

    if (header->input != NULL) {
        header->state = HeaderSink::DATA;
        istream_handler_set_direct(header->input,
                                   header->output.handler_direct);
    } else
        /* we have been closed meanwhile; bail out */
        consumed = 0;

    return consumed;
}

static size_t
header_consume_size(HeaderSink *header,
                    const void *data, size_t length)
{
    assert(header->position < sizeof(header->size_buffer));

    if (length > sizeof(header->size_buffer) - header->position)
        length = sizeof(header->size_buffer) - header->position;

    memcpy(header->size_buffer + header->position, data, length);
    header->position += length;

    if (header->position < sizeof(header->size_buffer))
        return length;

    const void *size_buffer = header->size_buffer;
        const uint32_t *size_p = (const uint32_t *)size_buffer;
    header->size = FromBE32(*size_p);
    if (header->size > 0x100000) {
        /* header too large */
        header->async_operation.Finished();
        istream_close_handler(header->input);

        GError *error =
            g_error_new_literal(sink_header_quark(), 0,
                                "header is too large");
        header->handler->error(error, header->handler_ctx);
        istream_deinit(&header->output);
        return 0;
    }

    if (header->size > 0) {
        header->buffer = (unsigned char *)
            p_malloc(header->output.pool, header->size);
        header->state = HeaderSink::HEADER;
        header->position = 0;
    } else {
        /* header empty: don't allocate, invoke callback now */

        header->buffer = NULL;

        length = header_invoke_callback(header, length);
    }

    return length;
}

static size_t
header_consume_header(HeaderSink *header,
                      const void *data, size_t length)
{
    size_t nbytes = header->size - header->position;

    assert(header->position < header->size);

    if (nbytes > length)
        nbytes = length;

    memcpy(header->buffer + header->position, data, nbytes);
    header->position += nbytes;

    if (header->position < header->size)
        return nbytes;

    return header_invoke_callback(header, nbytes);
}

/*
 * istream handler
 *
 */

static size_t
sink_header_input_data(const void *data0, size_t length, void *ctx)
{
    auto *header = (HeaderSink *)ctx;
    const unsigned char *data = (const unsigned char *)data0;
    size_t consumed = 0, nbytes;

    if (header->state == HeaderSink::DATA)
        return istream_invoke_data(&header->output, data, length);

    if (header->state == HeaderSink::SIZE) {
        header->pending = 0; /* just in case the callback is invoked */

        consumed = header_consume_size(header, data, length);
        if (consumed == 0)
            return 0;

        if (consumed == length)
            return length;

        data += consumed;
        length -= consumed;
    }

    if (header->state == HeaderSink::HEADER) {
        header->pending = consumed; /* just in case the callback is invoked */

        nbytes = header_consume_header(header, data, length);
        if (nbytes == 0)
            return 0;

        consumed += nbytes;
        if (consumed == length)
            return length;

        data += nbytes;
        length -= nbytes;
    }

    assert(consumed > 0);

    if (header->state == HeaderSink::DATA && length > 0) {
        const ScopePoolRef ref(*header->output.pool TRACE_ARGS);

        nbytes = istream_invoke_data(&header->output, data, length);
        if (nbytes == 0 && header->input == NULL)
            consumed = 0;
        else
            consumed += nbytes;
    }

    return consumed;
}

static ssize_t
sink_header_input_direct(FdType type, int fd, size_t max_length, void *ctx)
{
    auto *header = (HeaderSink *)ctx;

    assert(header->state == HeaderSink::DATA);

    return istream_invoke_direct(&header->output, type, fd, max_length);
}

static void
sink_header_input_eof(void *ctx)
{
    auto *header = (HeaderSink *)ctx;

    switch (header->state) {
        GError *error;

    case HeaderSink::SIZE:
    case HeaderSink::HEADER:
        header->async_operation.Finished();

        error = g_error_new_literal(sink_header_quark(), 0,
                                    "premature end of file");
        header->handler->error(error, header->handler_ctx);
        istream_deinit(&header->output);
        break;

    case HeaderSink::CALLBACK:
        assert(false);
        gcc_unreachable();

    case HeaderSink::DATA:
        istream_deinit_eof(&header->output);
        break;
    }
}

static void
sink_header_input_abort(GError *error, void *ctx)
{
    auto *header = (HeaderSink *)ctx;

    switch (header->state) {
    case HeaderSink::SIZE:
    case HeaderSink::HEADER:
        header->async_operation.Finished();
        header->handler->error(error, header->handler_ctx);
        istream_deinit(&header->output);
        break;

    case HeaderSink::CALLBACK:
        assert(false);
        gcc_unreachable();

    case HeaderSink::DATA:
        istream_deinit_abort(&header->output, error);
        break;
    }
}

static const struct istream_handler sink_header_input_handler = {
    .data = sink_header_input_data,
    .direct = sink_header_input_direct,
    .eof = sink_header_input_eof,
    .abort = sink_header_input_abort,
};

/*
 * istream implementation
 *
 */

static inline HeaderSink *
istream_to_header(struct istream *istream)
{
    return &ContainerCast2(*istream, &HeaderSink::output);
}

static off_t
sink_header_available(struct istream *istream, bool partial)
{
    HeaderSink *header = istream_to_header(istream);
    off_t available = istream_available(header->input, partial);

    if (available >= 0 && header->state == HeaderSink::CALLBACK) {
        if (available < (off_t)header->pending) {
            assert(partial);

            return -1;
        }

        available -= header->pending;
    }

    return available;
}

static void
sink_header_read(struct istream *istream)
{
    HeaderSink *header = istream_to_header(istream);

    if (header->state == HeaderSink::CALLBACK)
        /* workaround: when invoking the callback from the data()
           handler, it would be illegal to call header->input again */
        return;

    istream_handler_set_direct(header->input, header->output.handler_direct);
    istream_read(header->input);
}

static void
sink_header_close(struct istream *istream)
{
    HeaderSink *header = istream_to_header(istream);

    istream_free_handler(&header->input);
    istream_deinit(&header->output);
}

static const struct istream_class istream_sink = {
    .available = sink_header_available,
    .read = sink_header_read,
    .close = sink_header_close,
};

/*
 * async operation
 *
 */

static HeaderSink *
async_to_sink_header(struct async_operation *ao)
{
    return &ContainerCast2(*ao, &HeaderSink::async_operation);
}

static void
sink_header_abort(struct async_operation *ao)
{
    HeaderSink *header = async_to_sink_header(ao);

    istream_close_handler(header->input);
    istream_deinit(&header->output);
}

static const struct async_operation_class sink_header_operation = {
    .abort = sink_header_abort,
};

/*
 * constructor
 *
 */

void
sink_header_new(struct pool *pool, struct istream *input,
                const struct sink_header_handler *handler, void *ctx,
                struct async_operation_ref *async_ref)
{
    assert(input != NULL);
    assert(!istream_has_handler(input));
    assert(handler != NULL);
    assert(handler->done != NULL);
    assert(handler->error != NULL);

    auto header = NewFromPool<HeaderSink>(*pool);
    istream_init(&header->output, &istream_sink, pool);

    istream_assign_handler(&header->input, input,
                           &sink_header_input_handler, header,
                           0);

    header->state = HeaderSink::SIZE;
    header->position = 0;
    header->handler = handler;
    header->handler_ctx = ctx;

    header->async_operation.Init(sink_header_operation);
    async_ref->Set(header->async_operation);
}
<commit_msg>sink_header: move functions into the struct<commit_after>/*
 * This istream filter reads a 32 bit header size from the stream,
 * reads it into a buffer and invokes a callback with the tail of the
 * stream.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#include "sink_header.hxx"
#include "pool.hxx"
#include "istream_internal.hxx"
#include "async.hxx"
#include "util/Cast.hxx"
#include "util/ByteOrder.hxx"

#include <glib.h>

#include <assert.h>
#include <string.h>
#include <stdint.h>

struct HeaderSink {
    struct istream output;

    enum {
        SIZE, HEADER, CALLBACK, DATA
    } state;

    struct istream *input;

    unsigned char size_buffer[4];

    unsigned char *buffer;
    size_t size, position;

    /**
     * How much data of the input is pending to be consumed?  Only
     * valid while state==CALLBACK.
     */
    size_t pending;

    const struct sink_header_handler *handler;
    void *handler_ctx;

    struct async_operation async_operation;

    size_t InvokeCallback(size_t consumed);

    size_t ConsumeSize(const void *data, size_t length);
    size_t ConsumeHeader(const void *data, size_t length);
};

static GQuark
sink_header_quark(void)
{
    return g_quark_from_static_string("sink_header");
}

size_t
HeaderSink::InvokeCallback(size_t consumed)
{
    assert(state == SIZE || state == HEADER);

    async_operation.Finished();

    const ScopePoolRef ref(*output.pool TRACE_ARGS);

    /* the base value has been set by sink_header_input_data() */
    pending += consumed;

    state = CALLBACK;
    handler->done(buffer, size,
                  &output,
                  handler_ctx);

    if (input != NULL) {
        state = DATA;
        istream_handler_set_direct(input,
                                   output.handler_direct);
    } else
        /* we have been closed meanwhile; bail out */
        consumed = 0;

    return consumed;
}

inline size_t
HeaderSink::ConsumeSize(const void *data, size_t length)
{
    assert(position < sizeof(size_buffer));

    if (length > sizeof(size_buffer) - position)
        length = sizeof(size_buffer) - position;

    memcpy(size_buffer + position, data, length);
    position += length;

    if (position < sizeof(size_buffer))
        return length;

    const uint32_t *size_p = (const uint32_t *)(const void *)size_buffer;
    size = FromBE32(*size_p);
    if (size > 0x100000) {
        /* header too large */
        async_operation.Finished();
        istream_close_handler(input);

        GError *error =
            g_error_new_literal(sink_header_quark(), 0,
                                "header is too large");
        handler->error(error, handler_ctx);
        istream_deinit(&output);
        return 0;
    }

    if (size > 0) {
        buffer = (unsigned char *)
            p_malloc(output.pool, size);
        state = HeaderSink::HEADER;
        position = 0;
    } else {
        /* header empty: don't allocate, invoke callback now */

        buffer = nullptr;

        length = InvokeCallback(length);
    }

    return length;
}

inline size_t
HeaderSink::ConsumeHeader(const void *data, size_t length)
{
    size_t nbytes = size - position;

    assert(position < size);

    if (nbytes > length)
        nbytes = length;

    memcpy(buffer + position, data, nbytes);
    position += nbytes;

    if (position < size)
        return nbytes;

    return InvokeCallback(nbytes);
}

/*
 * istream handler
 *
 */

static size_t
sink_header_input_data(const void *data0, size_t length, void *ctx)
{
    auto *header = (HeaderSink *)ctx;
    const unsigned char *data = (const unsigned char *)data0;
    size_t consumed = 0, nbytes;

    if (header->state == HeaderSink::DATA)
        return istream_invoke_data(&header->output, data, length);

    if (header->state == HeaderSink::SIZE) {
        header->pending = 0; /* just in case the callback is invoked */

        consumed = header->ConsumeSize(data, length);
        if (consumed == 0)
            return 0;

        if (consumed == length)
            return length;

        data += consumed;
        length -= consumed;
    }

    if (header->state == HeaderSink::HEADER) {
        header->pending = consumed; /* just in case the callback is invoked */

        nbytes = header->ConsumeHeader(data, length);
        if (nbytes == 0)
            return 0;

        consumed += nbytes;
        if (consumed == length)
            return length;

        data += nbytes;
        length -= nbytes;
    }

    assert(consumed > 0);

    if (header->state == HeaderSink::DATA && length > 0) {
        const ScopePoolRef ref(*header->output.pool TRACE_ARGS);

        nbytes = istream_invoke_data(&header->output, data, length);
        if (nbytes == 0 && header->input == NULL)
            consumed = 0;
        else
            consumed += nbytes;
    }

    return consumed;
}

static ssize_t
sink_header_input_direct(FdType type, int fd, size_t max_length, void *ctx)
{
    auto *header = (HeaderSink *)ctx;

    assert(header->state == HeaderSink::DATA);

    return istream_invoke_direct(&header->output, type, fd, max_length);
}

static void
sink_header_input_eof(void *ctx)
{
    auto *header = (HeaderSink *)ctx;

    switch (header->state) {
        GError *error;

    case HeaderSink::SIZE:
    case HeaderSink::HEADER:
        header->async_operation.Finished();

        error = g_error_new_literal(sink_header_quark(), 0,
                                    "premature end of file");
        header->handler->error(error, header->handler_ctx);
        istream_deinit(&header->output);
        break;

    case HeaderSink::CALLBACK:
        assert(false);
        gcc_unreachable();

    case HeaderSink::DATA:
        istream_deinit_eof(&header->output);
        break;
    }
}

static void
sink_header_input_abort(GError *error, void *ctx)
{
    auto *header = (HeaderSink *)ctx;

    switch (header->state) {
    case HeaderSink::SIZE:
    case HeaderSink::HEADER:
        header->async_operation.Finished();
        header->handler->error(error, header->handler_ctx);
        istream_deinit(&header->output);
        break;

    case HeaderSink::CALLBACK:
        assert(false);
        gcc_unreachable();

    case HeaderSink::DATA:
        istream_deinit_abort(&header->output, error);
        break;
    }
}

static const struct istream_handler sink_header_input_handler = {
    .data = sink_header_input_data,
    .direct = sink_header_input_direct,
    .eof = sink_header_input_eof,
    .abort = sink_header_input_abort,
};

/*
 * istream implementation
 *
 */

static inline HeaderSink *
istream_to_header(struct istream *istream)
{
    return &ContainerCast2(*istream, &HeaderSink::output);
}

static off_t
sink_header_available(struct istream *istream, bool partial)
{
    HeaderSink *header = istream_to_header(istream);
    off_t available = istream_available(header->input, partial);

    if (available >= 0 && header->state == HeaderSink::CALLBACK) {
        if (available < (off_t)header->pending) {
            assert(partial);

            return -1;
        }

        available -= header->pending;
    }

    return available;
}

static void
sink_header_read(struct istream *istream)
{
    HeaderSink *header = istream_to_header(istream);

    if (header->state == HeaderSink::CALLBACK)
        /* workaround: when invoking the callback from the data()
           handler, it would be illegal to call header->input again */
        return;

    istream_handler_set_direct(header->input, header->output.handler_direct);
    istream_read(header->input);
}

static void
sink_header_close(struct istream *istream)
{
    HeaderSink *header = istream_to_header(istream);

    istream_free_handler(&header->input);
    istream_deinit(&header->output);
}

static const struct istream_class istream_sink = {
    .available = sink_header_available,
    .read = sink_header_read,
    .close = sink_header_close,
};

/*
 * async operation
 *
 */

static HeaderSink *
async_to_sink_header(struct async_operation *ao)
{
    return &ContainerCast2(*ao, &HeaderSink::async_operation);
}

static void
sink_header_abort(struct async_operation *ao)
{
    HeaderSink *header = async_to_sink_header(ao);

    istream_close_handler(header->input);
    istream_deinit(&header->output);
}

static const struct async_operation_class sink_header_operation = {
    .abort = sink_header_abort,
};

/*
 * constructor
 *
 */

void
sink_header_new(struct pool *pool, struct istream *input,
                const struct sink_header_handler *handler, void *ctx,
                struct async_operation_ref *async_ref)
{
    assert(input != NULL);
    assert(!istream_has_handler(input));
    assert(handler != NULL);
    assert(handler->done != NULL);
    assert(handler->error != NULL);

    auto header = NewFromPool<HeaderSink>(*pool);
    istream_init(&header->output, &istream_sink, pool);

    istream_assign_handler(&header->input, input,
                           &sink_header_input_handler, header,
                           0);

    header->state = HeaderSink::SIZE;
    header->position = 0;
    header->handler = handler;
    header->handler_ctx = ctx;

    header->async_operation.Init(sink_header_operation);
    async_ref->Set(header->async_operation);
}
<|endoftext|>


<commit_before>/*=========================================================================

  Program:   Visualization Toolkit
  Module:    vtkOpenGLTexture.cxx
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

Copyright (c) 1993-1998 Ken Martin, Will Schroeder, Bill Lorensen.

This software is copyrighted by Ken Martin, Will Schroeder and Bill Lorensen.
The following terms apply to all files associated with the software unless
explicitly disclaimed in individual files. This copyright specifically does
not apply to the related textbook "The Visualization Toolkit" ISBN
013199837-4 published by Prentice Hall which is covered by its own copyright.

The authors hereby grant permission to use, copy, and distribute this
software and its documentation for any purpose, provided that existing
copyright notices are retained in all copies and that this notice is included
verbatim in any distributions. Additionally, the authors grant permission to
modify this software and its documentation for any purpose, provided that
such modifications are not distributed without the explicit consent of the
authors and that existing copyright notices are retained in all copies. Some
of the algorithms implemented by this software are patented, observe all
applicable patent law.

IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT
OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY DERIVATIVES THEREOF,
EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES, INCLUDING,
BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
PARTICULAR PURPOSE, AND NON-INFRINGEMENT.  THIS SOFTWARE IS PROVIDED ON AN
"AS IS" BASIS, AND THE AUTHORS AND DISTRIBUTORS HAVE NO OBLIGATION TO PROVIDE
MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.

=========================================================================*/
#include <math.h>
#include <string.h>

#include "vtkOpenGLRenderer.h"
#include "vtkOpenGLTexture.h"
#include <GL/gl.h>

// shared increasing counter
long vtkOpenGLTexture::GlobalIndex = 0;

// Initializes an instance, generates a unique index.
vtkOpenGLTexture::vtkOpenGLTexture()
{
  this->Index = 0;
}

// Implement base class method.
void vtkOpenGLTexture::Load(vtkRenderer *vtkNotUsed(ren))
{
  GLenum format = GL_LUMINANCE;

  // need to reload the texture
  if (this->GetMTime() > this->LoadTime.GetMTime() ||
      this->Input->GetMTime() > this->LoadTime.GetMTime() ||
      (this->GetLookupTable() && this->GetLookupTable()->GetMTime () >
       this->LoadTime.GetMTime()))
    {
    int bytesPerPixel;
    int *size;
    vtkScalars *scalars;
    unsigned char *dataPtr;
    int rowLength;
    unsigned char *resultData=NULL;
    int xsize, ysize;
    unsigned short xs,ys;
    GLuint tempIndex;

    // get some info
    size = this->Input->GetDimensions();
    scalars = (this->Input->GetPointData())->GetScalars();

    // make sure scalars are non null
    if (!scalars)
      {
      vtkErrorMacro(<< "No scalar values found for texture input!");
      return;
      }

    bytesPerPixel = scalars->GetNumberOfComponents();

    // make sure using unsigned char data of color scalars type
    if (this->MapColorScalarsThroughLookupTable ||
       scalars->GetDataType() != VTK_UNSIGNED_CHAR )
      {
      dataPtr = this->MapScalarsToColors (scalars);
      bytesPerPixel = 4;
      }
    else
      {
      dataPtr = ((vtkUnsignedCharArray *)scalars->GetData())->GetPointer(0);
      }

    // we only support 2d texture maps right now
    // so one of the three sizes must be 1, but it
    // could be any of them, so lets find it
    if (size[0] == 1)
      {
      xsize = size[1]; ysize = size[2];
      }
    else
      {
      xsize = size[0];
      if (size[1] == 1)
        {
        ysize = size[2];
        }
      else
        {
        ysize = size[1];
        if (size[2] != 1)
          {
          vtkErrorMacro(<< "3D texture maps currently are not supported!");
          return;
          }
        }
      }

    // xsize and ysize must be a power of 2 in OpenGL
    xs = (unsigned short)xsize;
    ys = (unsigned short)ysize;
    while (!(xs & 0x01))
      {
      xs = xs >> 1;
      }
    while (!(ys & 0x01))
      {
      ys = ys >> 1;
      }
    if ((xs > 1)||(ys > 1))
      {
      vtkWarningMacro(<< "Resampling texture to power of two for OpenGL");
      resultData = this->ResampleToPowerOfTwo(xsize, ysize, dataPtr,
                                              bytesPerPixel);
      }

    // format the data so that it can be sent to opengl
    // each row must be a multiple of 4 bytes in length
    // the best idea is to make your size a multiple of 4
    // so that this conversion will never be done.
    rowLength = ((xsize*bytesPerPixel +3 )/4)*4;
    if (rowLength == xsize*bytesPerPixel)
      {
      if ( resultData == NULL )
        {
        resultData = dataPtr;
        }
      }
    else
      {
      int col;
      unsigned char *src,*dest;
      int srcLength;

      srcLength = xsize*bytesPerPixel;
      resultData = new unsigned char [rowLength*ysize];

      src = dataPtr;
      dest = resultData;

      for (col = 0; col < ysize; col++)
        {
        memcpy(dest,src,srcLength);
        src += srcLength;
        dest += rowLength;
        }
      }

    // define a display list for this texture
    // free any old display lists
    if (this->Index)
      {
#ifdef GL_VERSION_1_1
      tempIndex = this->Index;
      glDeleteTextures(1, &tempIndex);
#else
      glDeleteLists(this->Index,1);
#endif
      this->Index = 0;
      }
    // get a unique display list id
#ifdef GL_VERSION_1_1
    glGenTextures(1, &tempIndex);
    this->Index = tempIndex;
    glBindTexture(GL_TEXTURE_2D, this->Index);
#else
    this->Index = glGenLists(1);
    glDeleteLists ((GLuint) this->Index, (GLsizei) 0);
    glNewList ((GLuint) this->Index, GL_COMPILE);
#endif
    if (this->Interpolate)
      {
      glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
                       GL_LINEAR);
      glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
                       GL_LINEAR );
      }
    else
      {
      glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST );
      glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST );
      }
    if (this->Repeat)
      {
      glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,     GL_REPEAT );
      glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,     GL_REPEAT );
      }
    else
      {
      glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,     GL_CLAMP );
      glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,     GL_CLAMP );
      }
    switch (bytesPerPixel)
      {
      case 1: format = GL_LUMINANCE; break;
      case 2: format = GL_LUMINANCE_ALPHA; break;
      case 3: format = GL_RGB; break;
      case 4: format = GL_RGBA; break;
      }
    glTexImage2D( GL_TEXTURE_2D, 0 , bytesPerPixel,
                  xsize, ysize, 0, format,
                  GL_UNSIGNED_BYTE, (const GLvoid *)resultData );
#ifndef GL_VERSION_1_1
    glEndList ();
#endif
    // modify the load time to the current time
    this->LoadTime.Modified();

    // free memory
    if (resultData != dataPtr)
      {
      delete [] resultData;
      }
    }

  // execute the display list that uses creates the texture
#ifdef GL_VERSION_1_1
  glBindTexture(GL_TEXTURE_2D, this->Index);
#else
  glCallList ((GLuint) this->Index);
#endif

  // don't accept fragments if they have zero opacity. this will stop the
  // zbuffer from be blocked by totally transparent texture fragments.
  glAlphaFunc (GL_GREATER, (GLclampf) 0);
  glEnable (GL_ALPHA_TEST);

  // now bind it
  glEnable(GL_TEXTURE_2D);
}

void vtkOpenGLTexture::ReleaseGraphicsResources(vtkWindow *renWin)
{
  GLuint tempIndex;

#ifdef GL_VERSION_1_1
  tempIndex = this->Index;
  glDeleteTextures(1, &tempIndex);
#else
  glDeleteLists(this->Index,1);
#endif
  this->Index = 0;
}

static int FindPowerOfTwo(int i)
{
  int size;

  for ( i--, size=1; i > 0; size*=2 )
    {
    i /= 2;
    }

  return size;
}

// Creates resampled unsigned char texture map that is a power of two in bith x and y.
unsigned char *vtkOpenGLTexture::ResampleToPowerOfTwo(int &xs, int &ys, unsigned char *dptr,
                                                      int bpp)
{
  unsigned char *tptr, *p, *p1, *p2, *p3, *p4;
  int xsize, ysize, i, j, k, jOffset, iIdx, jIdx;
  float pcoords[3], hx, hy, rm, sm, w0, w1, w2, w3;

  xsize = FindPowerOfTwo(xs);
  ysize = FindPowerOfTwo(ys);

  hx = (float)(xs - 1.0) / (xsize - 1.0);
  hy = (float)(ys - 1.0) / (ysize - 1.0);

  tptr = p = new unsigned char[xsize*ysize*bpp];

  //Resample from the previous image. Compute parametric coordinates and interpolate
  for (j=0; j < ysize; j++)
    {
    pcoords[1] = j*hy;

    jIdx = (int)pcoords[1];
    if ( jIdx >= (ys-1) ) //make sure to interpolate correctly at edge
      {
      jIdx = ys - 2;
      pcoords[1] = 1.0;
      }
    else
      {
      pcoords[1] = pcoords[1] - jIdx;
      }
    jOffset = jIdx*xs;
    sm = 1.0 - pcoords[1];

    for (i=0; i < xsize; i++)
      {
      pcoords[0] = i*hx;
      iIdx = (int)pcoords[0];
      if ( iIdx >= (xs-1) )
        {
        iIdx = xs - 2;
        pcoords[0] = 1.0;
        }
      else
        {
        pcoords[0] = pcoords[0] - iIdx;
        }
      rm = 1.0 - pcoords[0];

      // Get pointers to 4 surrounding pixels
      p1 = dptr + bpp*(iIdx + jOffset);
      p2 = p1 + bpp;
      p3 = p1 + bpp*xs;
      p4 = p3 + bpp;

      // Compute interpolation weights interpolate components
      w0 = rm*sm;
      w1 = pcoords[0]*sm;
      w2 = rm*pcoords[1];
      w3 = pcoords[0]*pcoords[1];
      for (k=0; k < bpp; k++)
        {
        *p++ = p1[k]*w0 + p2[k]*w1 + p3[k]*w2 + p4[k]*w3;
        }
      }
    }

  xs = xsize;
  ys = ysize;

  return tptr;
}
<commit_msg>ENH:Use debug macro<commit_after>/*=========================================================================

  Program:   Visualization Toolkit
  Module:    vtkOpenGLTexture.cxx
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

Copyright (c) 1993-1998 Ken Martin, Will Schroeder, Bill Lorensen.

This software is copyrighted by Ken Martin, Will Schroeder and Bill Lorensen.
The following terms apply to all files associated with the software unless
explicitly disclaimed in individual files. This copyright specifically does
not apply to the related textbook "The Visualization Toolkit" ISBN
013199837-4 published by Prentice Hall which is covered by its own copyright.

The authors hereby grant permission to use, copy, and distribute this
software and its documentation for any purpose, provided that existing
copyright notices are retained in all copies and that this notice is included
verbatim in any distributions. Additionally, the authors grant permission to
modify this software and its documentation for any purpose, provided that
such modifications are not distributed without the explicit consent of the
authors and that existing copyright notices are retained in all copies. Some
of the algorithms implemented by this software are patented, observe all
applicable patent law.

IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT
OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY DERIVATIVES THEREOF,
EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES, INCLUDING,
BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
PARTICULAR PURPOSE, AND NON-INFRINGEMENT.  THIS SOFTWARE IS PROVIDED ON AN
"AS IS" BASIS, AND THE AUTHORS AND DISTRIBUTORS HAVE NO OBLIGATION TO PROVIDE
MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.

=========================================================================*/
#include <math.h>
#include <string.h>

#include "vtkOpenGLRenderer.h"
#include "vtkOpenGLTexture.h"
#include <GL/gl.h>

// shared increasing counter
long vtkOpenGLTexture::GlobalIndex = 0;

// Initializes an instance, generates a unique index.
vtkOpenGLTexture::vtkOpenGLTexture()
{
  this->Index = 0;
}

// Implement base class method.
void vtkOpenGLTexture::Load(vtkRenderer *vtkNotUsed(ren))
{
  GLenum format = GL_LUMINANCE;

  // need to reload the texture
  if (this->GetMTime() > this->LoadTime.GetMTime() ||
      this->Input->GetMTime() > this->LoadTime.GetMTime() ||
      (this->GetLookupTable() && this->GetLookupTable()->GetMTime () >
       this->LoadTime.GetMTime()))
    {
    int bytesPerPixel;
    int *size;
    vtkScalars *scalars;
    unsigned char *dataPtr;
    int rowLength;
    unsigned char *resultData=NULL;
    int xsize, ysize;
    unsigned short xs,ys;
    GLuint tempIndex;

    // get some info
    size = this->Input->GetDimensions();
    scalars = (this->Input->GetPointData())->GetScalars();

    // make sure scalars are non null
    if (!scalars)
      {
      vtkErrorMacro(<< "No scalar values found for texture input!");
      return;
      }

    bytesPerPixel = scalars->GetNumberOfComponents();

    // make sure using unsigned char data of color scalars type
    if (this->MapColorScalarsThroughLookupTable ||
       scalars->GetDataType() != VTK_UNSIGNED_CHAR )
      {
      dataPtr = this->MapScalarsToColors (scalars);
      bytesPerPixel = 4;
      }
    else
      {
      dataPtr = ((vtkUnsignedCharArray *)scalars->GetData())->GetPointer(0);
      }

    // we only support 2d texture maps right now
    // so one of the three sizes must be 1, but it
    // could be any of them, so lets find it
    if (size[0] == 1)
      {
      xsize = size[1]; ysize = size[2];
      }
    else
      {
      xsize = size[0];
      if (size[1] == 1)
        {
        ysize = size[2];
        }
      else
        {
        ysize = size[1];
        if (size[2] != 1)
          {
          vtkErrorMacro(<< "3D texture maps currently are not supported!");
          return;
          }
        }
      }

    // xsize and ysize must be a power of 2 in OpenGL
    xs = (unsigned short)xsize;
    ys = (unsigned short)ysize;
    while (!(xs & 0x01))
      {
      xs = xs >> 1;
      }
    while (!(ys & 0x01))
      {
      ys = ys >> 1;
      }
    if ((xs > 1)||(ys > 1))
      {
      vtkDebugMacro(<< "Resampling texture to power of two for OpenGL");
      resultData = this->ResampleToPowerOfTwo(xsize, ysize, dataPtr,
                                              bytesPerPixel);
      }

    // format the data so that it can be sent to opengl
    // each row must be a multiple of 4 bytes in length
    // the best idea is to make your size a multiple of 4
    // so that this conversion will never be done.
    rowLength = ((xsize*bytesPerPixel +3 )/4)*4;
    if (rowLength == xsize*bytesPerPixel)
      {
      if ( resultData == NULL )
        {
        resultData = dataPtr;
        }
      }
    else
      {
      int col;
      unsigned char *src,*dest;
      int srcLength;

      srcLength = xsize*bytesPerPixel;
      resultData = new unsigned char [rowLength*ysize];

      src = dataPtr;
      dest = resultData;

      for (col = 0; col < ysize; col++)
        {
        memcpy(dest,src,srcLength);
        src += srcLength;
        dest += rowLength;
        }
      }

    // define a display list for this texture
    // free any old display lists
    if (this->Index)
      {
#ifdef GL_VERSION_1_1
      tempIndex = this->Index;
      glDeleteTextures(1, &tempIndex);
#else
      glDeleteLists(this->Index,1);
#endif
      this->Index = 0;
      }
    // get a unique display list id
#ifdef GL_VERSION_1_1
    glGenTextures(1, &tempIndex);
    this->Index = tempIndex;
    glBindTexture(GL_TEXTURE_2D, this->Index);
#else
    this->Index = glGenLists(1);
    glDeleteLists ((GLuint) this->Index, (GLsizei) 0);
    glNewList ((GLuint) this->Index, GL_COMPILE);
#endif
    if (this->Interpolate)
      {
      glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
                       GL_LINEAR);
      glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
                       GL_LINEAR );
      }
    else
      {
      glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST );
      glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST );
      }
    if (this->Repeat)
      {
      glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,     GL_REPEAT );
      glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,     GL_REPEAT );
      }
    else
      {
      glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,     GL_CLAMP );
      glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,     GL_CLAMP );
      }
    switch (bytesPerPixel)
      {
      case 1: format = GL_LUMINANCE; break;
      case 2: format = GL_LUMINANCE_ALPHA; break;
      case 3: format = GL_RGB; break;
      case 4: format = GL_RGBA; break;
      }
    glTexImage2D( GL_TEXTURE_2D, 0 , bytesPerPixel,
                  xsize, ysize, 0, format,
                  GL_UNSIGNED_BYTE, (const GLvoid *)resultData );
#ifndef GL_VERSION_1_1
    glEndList ();
#endif
    // modify the load time to the current time
    this->LoadTime.Modified();

    // free memory
    if (resultData != dataPtr)
      {
      delete [] resultData;
      }
    }

  // execute the display list that uses creates the texture
#ifdef GL_VERSION_1_1
  glBindTexture(GL_TEXTURE_2D, this->Index);
#else
  glCallList ((GLuint) this->Index);
#endif

  // don't accept fragments if they have zero opacity. this will stop the
  // zbuffer from be blocked by totally transparent texture fragments.
  glAlphaFunc (GL_GREATER, (GLclampf) 0);
  glEnable (GL_ALPHA_TEST);

  // now bind it
  glEnable(GL_TEXTURE_2D);
}

void vtkOpenGLTexture::ReleaseGraphicsResources(vtkWindow *renWin)
{
  GLuint tempIndex;

#ifdef GL_VERSION_1_1
  tempIndex = this->Index;
  glDeleteTextures(1, &tempIndex);
#else
  glDeleteLists(this->Index,1);
#endif
  this->Index = 0;
}

static int FindPowerOfTwo(int i)
{
  int size;

  for ( i--, size=1; i > 0; size*=2 )
    {
    i /= 2;
    }

  return size;
}

// Creates resampled unsigned char texture map that is a power of two in bith x and y.
unsigned char *vtkOpenGLTexture::ResampleToPowerOfTwo(int &xs, int &ys, unsigned char *dptr,
                                                      int bpp)
{
  unsigned char *tptr, *p, *p1, *p2, *p3, *p4;
  int xsize, ysize, i, j, k, jOffset, iIdx, jIdx;
  float pcoords[3], hx, hy, rm, sm, w0, w1, w2, w3;

  xsize = FindPowerOfTwo(xs);
  ysize = FindPowerOfTwo(ys);

  hx = (float)(xs - 1.0) / (xsize - 1.0);
  hy = (float)(ys - 1.0) / (ysize - 1.0);

  tptr = p = new unsigned char[xsize*ysize*bpp];

  //Resample from the previous image. Compute parametric coordinates and interpolate
  for (j=0; j < ysize; j++)
    {
    pcoords[1] = j*hy;

    jIdx = (int)pcoords[1];
    if ( jIdx >= (ys-1) ) //make sure to interpolate correctly at edge
      {
      jIdx = ys - 2;
      pcoords[1] = 1.0;
      }
    else
      {
      pcoords[1] = pcoords[1] - jIdx;
      }
    jOffset = jIdx*xs;
    sm = 1.0 - pcoords[1];

    for (i=0; i < xsize; i++)
      {
      pcoords[0] = i*hx;
      iIdx = (int)pcoords[0];
      if ( iIdx >= (xs-1) )
        {
        iIdx = xs - 2;
        pcoords[0] = 1.0;
        }
      else
        {
        pcoords[0] = pcoords[0] - iIdx;
        }
      rm = 1.0 - pcoords[0];

      // Get pointers to 4 surrounding pixels
      p1 = dptr + bpp*(iIdx + jOffset);
      p2 = p1 + bpp;
      p3 = p1 + bpp*xs;
      p4 = p3 + bpp;

      // Compute interpolation weights interpolate components
      w0 = rm*sm;
      w1 = pcoords[0]*sm;
      w2 = rm*pcoords[1];
      w3 = pcoords[0]*pcoords[1];
      for (k=0; k < bpp; k++)
        {
        *p++ = p1[k]*w0 + p2[k]*w1 + p3[k]*w2 + p4[k]*w3;
        }
      }
    }

  xs = xsize;
  ys = ysize;

  return tptr;
}
<|endoftext|>


<commit_before>// This file is a part of the IncludeOS unikernel - www.includeos.org
//
// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences
// and Alfred Bratterud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <kernel/os.hpp>

extern "C" void _start() __attribute__((visibility("hidden"))) __attribute__ ((noreturn));
extern "C" void _init_c_runtime();
extern "C" void _init_syscalls();

// enables Streaming SIMD Extensions
static void enableSSE(void)
{
  asm ("mov %cr0, %eax");
  asm ("and $0xFFFB,%ax");
  asm ("or  $0x2,   %ax");
  asm ("mov %eax, %cr0");

  asm ("mov %cr4, %eax");
  asm ("or  $0x600,%ax");
  asm ("mov %eax, %cr4");
}

extern "C" void kernel_start(uintptr_t magic, uintptr_t addr)  {

  // enable SSE extensions bitmask in CR4 register
  enableSSE();

  // Initialize system calls
  _init_syscalls();

  // Initialize stack-unwinder, call global constructors etc.
  _init_c_runtime();

  // Initialize OS including devices
  OS::start(magic, addr);
}
<commit_msg>kernel: save multiboot CLI string before strtab copy<commit_after>// This file is a part of the IncludeOS unikernel - www.includeos.org
//
// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences
// and Alfred Bratterud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <kernel/os.hpp>
#include <boot/multiboot.h>

#define MULTIBOOT_CMDLINE_LOC 0x3000

extern "C" void _start() __attribute__((visibility("hidden"))) __attribute__ ((noreturn));
extern "C" void _init_c_runtime();
extern "C" void _init_syscalls();

// enables Streaming SIMD Extensions
static void enableSSE(void)
{
  asm ("mov %cr0, %eax");
  asm ("and $0xFFFB,%ax");
  asm ("or  $0x2,   %ax");
  asm ("mov %eax, %cr0");

  asm ("mov %cr4, %eax");
  asm ("or  $0x600,%ax");
  asm ("mov %eax, %cr4");
}

extern "C" void kernel_start(uintptr_t magic, uintptr_t addr)  {

  // enable SSE extensions bitmask in CR4 register
  enableSSE();

  // Initialize system calls
  _init_syscalls();

  // Save multiboot string before symbols overwrite area after binary
  char* cmdline = reinterpret_cast<char*>(reinterpret_cast<multiboot_info_t*>(addr)->cmdline);
  strcpy(reinterpret_cast<char*>(MULTIBOOT_CMDLINE_LOC), cmdline);
  ((multiboot_info_t*) addr)->cmdline = MULTIBOOT_CMDLINE_LOC;

  // Initialize stack-unwinder, call global constructors etc.
  _init_c_runtime();

  // Initialize OS including devices
  OS::start(magic, addr);
}
<|endoftext|>


<commit_before>/**
 * @file tpot.cpp
 *
 * @brief Plugin to calculate potential temperature
 *
 * Created on: Nov 20, 2012
 * @author partio
 */

#include "tpot.h"
#include <iostream>
#include "plugin_factory.h"
#include "logger_factory.h"
#include <boost/lexical_cast.hpp>

#define HIMAN_AUXILIARY_INCLUDE

#include "fetcher.h"
#include "writer.h"
#include "neons.h"
#include "pcuda.h"

#undef HIMAN_AUXILIARY_INCLUDE

#ifdef DEBUG
#include "timer_factory.h"
#endif

using namespace std;
using namespace himan::plugin;

#include "cuda_extern.h"

tpot::tpot() : itsUseCuda(false), itsCudaDeviceCount(0)
{
	itsClearTextFormula = "Tp = Tk * pow((1000/P), 0.286)"; // Poissons equation

	itsLogger = std::unique_ptr<logger> (logger_factory::Instance()->GetLog("tpot"));

}

void tpot::Process(std::shared_ptr<const plugin_configuration> conf)
{

	shared_ptr<plugin::pcuda> c = dynamic_pointer_cast<plugin::pcuda> (plugin_factory::Instance()->Plugin("pcuda"));

	if (c->HaveCuda())
	{
		string msg = "I possess the powers of CUDA";

		if (!conf->UseCuda())
		{
			msg += ", but I won't use them";
		}
		else
		{
			msg += ", and I'm not afraid to use them";
			itsUseCuda = true;
		}

		itsLogger->Info(msg);

		itsCudaDeviceCount = c->DeviceCount();

	}

	// Get number of threads to use

	unsigned short threadCount = ThreadCount(conf->ThreadCount());

	if (conf->StatisticsEnabled())
	{
		conf->Statistics()->UsedThreadCount(threadCount);
		conf->Statistics()->UsedGPUCount(itsCudaDeviceCount);
	}

	boost::thread_group g;

	shared_ptr<info> targetInfo = conf->Info();

	/*
	 * Set target parameter to potential temperature
	 * - name TP-K
	 * - univ_id 8
	 * - grib2 descriptor 0'00'002
	 *
	 * We need to specify grib and querydata parameter information
	 * since we don't know which one will be the output format.
	 * (todo: we could check from conf but why bother?)
	 *
	 */

	vector<param> theParams;

	param theRequestedParam ("TP-K", 8);

	// GRIB 2

	theRequestedParam.GribDiscipline(0);
	theRequestedParam.GribCategory(0);
	theRequestedParam.GribParameter(2);

	// GRIB 1

	if (conf->OutputFileType() == kGRIB1)
	{
		shared_ptr<neons> n = dynamic_pointer_cast<neons> (plugin_factory::Instance()->Plugin("neons"));

		long parm_id = n->NeonsDB().GetGridParameterId(targetInfo->Producer().TableVersion(), theRequestedParam.Name());
		theRequestedParam.GribIndicatorOfParameter(parm_id);
		theRequestedParam.GribTableVersion(targetInfo->Producer().TableVersion());

	}

	theParams.push_back(theRequestedParam);

	targetInfo->Params(theParams);

	/*
	 * Create data structures.
	 */

	targetInfo->Create();

	/*
	 * Initialize parent class functions for dimension handling
	 */

	Dimension(conf->LeadingDimension());
	FeederInfo(shared_ptr<info> (new info(*targetInfo)));
	FeederInfo()->Param(theRequestedParam);

	/*
	 * Each thread will have a copy of the target info.
	 */

	vector<shared_ptr<info> > targetInfos;

	targetInfos.resize(threadCount);

	for (size_t i = 0; i < threadCount; i++)
	{

		itsLogger->Info("Thread " + boost::lexical_cast<string> (i + 1) + " starting");

		targetInfos[i] = shared_ptr<info> (new info(*targetInfo));

		boost::thread* t = new boost::thread(&tpot::Run,
											 this,
											 targetInfos[i],
											 conf,
											 i + 1);

		g.add_thread(t);

	}

	g.join_all();

	if (conf->FileWriteOption() == kSingleFile)
	{

		shared_ptr<writer> theWriter = dynamic_pointer_cast <writer> (plugin_factory::Instance()->Plugin("writer"));

		string theOutputFile = conf->ConfigurationFile();

		theWriter->ToFile(targetInfo, conf, theOutputFile);

	}
}

void tpot::Run(shared_ptr<info> myTargetInfo, shared_ptr<const plugin_configuration> conf, unsigned short threadIndex)
{

	while (AdjustLeadingDimension(myTargetInfo))
	{
		Calculate(myTargetInfo, conf, threadIndex);
	}

}

/*
 * Calculate()
 *
 * This function does the actual calculation.
 */

void tpot::Calculate(shared_ptr<info> myTargetInfo, shared_ptr<const plugin_configuration> conf, unsigned short threadIndex)
{

	shared_ptr<fetcher> theFetcher = dynamic_pointer_cast <fetcher> (plugin_factory::Instance()->Plugin("fetcher"));

	// Required source parameters

	param TParam ("T-K");
	param PParam ("P-PA");

	unique_ptr<logger> myThreadedLogger = std::unique_ptr<logger> (logger_factory::Instance()->GetLog("tpotThread #" + boost::lexical_cast<string> (threadIndex)));

	ResetNonLeadingDimension(myTargetInfo);

	myTargetInfo->FirstParam();

	while (AdjustNonLeadingDimension(myTargetInfo))
	{

		myThreadedLogger->Debug("Calculating time " + myTargetInfo->Time().ValidDateTime()->String("%Y%m%d%H") +
								" level " + boost::lexical_cast<string> (myTargetInfo->Level().Value()));

		double PScale = 1;
		double TBase = 0;

		// Source infos
		shared_ptr<info> TInfo;
		shared_ptr<info> PInfo;

		shared_ptr<NFmiGrid> PGrid;

		bool isPressureLevel = (myTargetInfo->Level().Type() == kPressure);

		try
		{

			TInfo = theFetcher->Fetch(conf,
											 myTargetInfo->Time(),
											 myTargetInfo->Level(),
											 TParam);

			if (!isPressureLevel)
			{
				// Source info for P
				PInfo = theFetcher->Fetch(conf,
										  myTargetInfo->Time(),
										  myTargetInfo->Level(),
										  PParam);

				if (PInfo->Param().Unit() == kPa)
				{
					PScale = 0.01;
				}

				PGrid = shared_ptr<NFmiGrid> (PInfo->Grid()->ToNewbaseGrid());
			}
		}
		catch (HPExceptionType e)
		{
			switch (e)
			{
				case kFileDataNotFound:
					itsLogger->Info("Skipping step " + boost::lexical_cast<string> (myTargetInfo->Time().Step()) + ", level " + boost::lexical_cast<string> (myTargetInfo->Level().Value()));
					myTargetInfo->Data()->Fill(kFloatMissing); // Fill data with missing value

					if (conf->StatisticsEnabled())
					{
						conf->Statistics()->AddToMissingCount(myTargetInfo->Grid()->Size());
						conf->Statistics()->AddToValueCount(myTargetInfo->Grid()->Size());
					}

					continue;
					break;

				default:
					throw runtime_error(ClassName() + ": Unable to proceed");
					break;
			}
		}

		unique_ptr<timer> processTimer = unique_ptr<timer> (timer_factory::Instance()->GetTimer());

		if (conf->StatisticsEnabled())
		{
			processTimer->Start();
		}

		if (TInfo->Param().Unit() == kC)
		{
			TBase = 273.15;
		}

		shared_ptr<NFmiGrid> targetGrid(myTargetInfo->Grid()->ToNewbaseGrid());
		shared_ptr<NFmiGrid> TGrid(TInfo->Grid()->ToNewbaseGrid());

		int missingCount = 0;
		int count = 0;

		assert(targetGrid->Size() == myTargetInfo->Data()->Size());

		bool equalGrids = (*myTargetInfo->Grid() == *TInfo->Grid() && (isPressureLevel || *myTargetInfo->Grid() == *PInfo->Grid()));

		string deviceType;

		if (itsUseCuda && equalGrids && threadIndex <= itsCudaDeviceCount)
		{
			deviceType = "GPU";

			size_t N = TGrid->Size();

			double* TPOut;

#ifdef HAVE_CUDA
			cudaMallocHost((void**) &TPOut, N*sizeof(double));
#else
			TPOut = new double[N];
#endif

			if (!isPressureLevel)
			{
				tpot_cuda::DoCuda(TInfo->Data()->Values(), TBase, PInfo->Data()->Values(), PScale, TPOut, N, 0, threadIndex-1);
			}
			else
			{
				tpot_cuda::DoCuda(TInfo->Data()->Values(), TBase, 0, 0, TPOut, N, myTargetInfo->Level().Value(), threadIndex-1);
			}

			myTargetInfo->Data()->Set(TPOut, N);

#ifdef HAVE_CUDA
			cudaFreeHost(TPOut);
#else
			delete [] TPOut;
#endif
		}
		else
		{

			deviceType = "CPU";

			myTargetInfo->ResetLocation();

			targetGrid->Reset();

			while (myTargetInfo->NextLocation() && targetGrid->Next())
			{
				count++;

				double T = kFloatMissing;
				double P = kFloatMissing;

				InterpolateToPoint(targetGrid, TGrid, equalGrids, T);

				if (isPressureLevel)
				{
					P = myTargetInfo->Level().Value();
				}
				else
				{
					InterpolateToPoint(targetGrid, PGrid, equalGrids, P);
				}

				if (T == kFloatMissing || P == kFloatMissing)
				{
					missingCount++;

					myTargetInfo->Value(kFloatMissing);
					continue;
				}

				double Tp = (T + TBase) * pow((1000 / (P * PScale)), 0.286);

				if (!myTargetInfo->Value(Tp))
				{
					throw runtime_error(ClassName() + ": Failed to set value to matrix");
				}
			}
		}

		if (conf->StatisticsEnabled())
		{
			processTimer->Stop();
			conf->Statistics()->AddToProcessingTime(processTimer->GetTime());

#ifdef DEBUG
			itsLogger->Debug("Calculation took " + boost::lexical_cast<string> (processTimer->GetTime()) + " microseconds on "  + deviceType);
#endif

			conf->Statistics()->AddToMissingCount(missingCount);
			conf->Statistics()->AddToValueCount(count);

		}

		/*
		 * Now we are done for this level
		 *
		 * Clone info-instance to writer since it might change our descriptor places
		 * */

		myThreadedLogger->Info("Missing values: " + boost::lexical_cast<string> (missingCount) + "/" + boost::lexical_cast<string> (count));

		if (conf->FileWriteOption() == kNeons || conf->FileWriteOption() == kMultipleFiles)
		{
			shared_ptr<writer> theWriter = dynamic_pointer_cast <writer> (plugin_factory::Instance()->Plugin("writer"));

			theWriter->ToFile(shared_ptr<info> (new info(*myTargetInfo)), conf);
		}
	}
}
<commit_msg>do no use old-style-cast<commit_after>/**
 * @file tpot.cpp
 *
 * @brief Plugin to calculate potential temperature
 *
 * Created on: Nov 20, 2012
 * @author partio
 */

#include "tpot.h"
#include <iostream>
#include "plugin_factory.h"
#include "logger_factory.h"
#include <boost/lexical_cast.hpp>

#define HIMAN_AUXILIARY_INCLUDE

#include "fetcher.h"
#include "writer.h"
#include "neons.h"
#include "pcuda.h"

#undef HIMAN_AUXILIARY_INCLUDE

#ifdef DEBUG
#include "timer_factory.h"
#endif

using namespace std;
using namespace himan::plugin;

#include "cuda_extern.h"

tpot::tpot() : itsUseCuda(false), itsCudaDeviceCount(0)
{
	itsClearTextFormula = "Tp = Tk * pow((1000/P), 0.286)"; // Poissons equation

	itsLogger = std::unique_ptr<logger> (logger_factory::Instance()->GetLog("tpot"));

}

void tpot::Process(std::shared_ptr<const plugin_configuration> conf)
{

	shared_ptr<plugin::pcuda> c = dynamic_pointer_cast<plugin::pcuda> (plugin_factory::Instance()->Plugin("pcuda"));

	if (c->HaveCuda())
	{
		string msg = "I possess the powers of CUDA";

		if (!conf->UseCuda())
		{
			msg += ", but I won't use them";
		}
		else
		{
			msg += ", and I'm not afraid to use them";
			itsUseCuda = true;
		}

		itsLogger->Info(msg);

		itsCudaDeviceCount = c->DeviceCount();

	}

	// Get number of threads to use

	unsigned short threadCount = ThreadCount(conf->ThreadCount());

	if (conf->StatisticsEnabled())
	{
		conf->Statistics()->UsedThreadCount(threadCount);
		conf->Statistics()->UsedGPUCount(itsCudaDeviceCount);
	}

	boost::thread_group g;

	shared_ptr<info> targetInfo = conf->Info();

	/*
	 * Set target parameter to potential temperature
	 * - name TP-K
	 * - univ_id 8
	 * - grib2 descriptor 0'00'002
	 *
	 * We need to specify grib and querydata parameter information
	 * since we don't know which one will be the output format.
	 * (todo: we could check from conf but why bother?)
	 *
	 */

	vector<param> theParams;

	param theRequestedParam ("TP-K", 8);

	// GRIB 2

	theRequestedParam.GribDiscipline(0);
	theRequestedParam.GribCategory(0);
	theRequestedParam.GribParameter(2);

	// GRIB 1

	if (conf->OutputFileType() == kGRIB1)
	{
		shared_ptr<neons> n = dynamic_pointer_cast<neons> (plugin_factory::Instance()->Plugin("neons"));

		long parm_id = n->NeonsDB().GetGridParameterId(targetInfo->Producer().TableVersion(), theRequestedParam.Name());
		theRequestedParam.GribIndicatorOfParameter(parm_id);
		theRequestedParam.GribTableVersion(targetInfo->Producer().TableVersion());

	}

	theParams.push_back(theRequestedParam);

	targetInfo->Params(theParams);

	/*
	 * Create data structures.
	 */

	targetInfo->Create();

	/*
	 * Initialize parent class functions for dimension handling
	 */

	Dimension(conf->LeadingDimension());
	FeederInfo(shared_ptr<info> (new info(*targetInfo)));
	FeederInfo()->Param(theRequestedParam);

	/*
	 * Each thread will have a copy of the target info.
	 */

	vector<shared_ptr<info> > targetInfos;

	targetInfos.resize(threadCount);

	for (size_t i = 0; i < threadCount; i++)
	{

		itsLogger->Info("Thread " + boost::lexical_cast<string> (i + 1) + " starting");

		targetInfos[i] = shared_ptr<info> (new info(*targetInfo));

		boost::thread* t = new boost::thread(&tpot::Run,
											 this,
											 targetInfos[i],
											 conf,
											 i + 1);

		g.add_thread(t);

	}

	g.join_all();

	if (conf->FileWriteOption() == kSingleFile)
	{

		shared_ptr<writer> theWriter = dynamic_pointer_cast <writer> (plugin_factory::Instance()->Plugin("writer"));

		string theOutputFile = conf->ConfigurationFile();

		theWriter->ToFile(targetInfo, conf, theOutputFile);

	}
}

void tpot::Run(shared_ptr<info> myTargetInfo, shared_ptr<const plugin_configuration> conf, unsigned short threadIndex)
{

	while (AdjustLeadingDimension(myTargetInfo))
	{
		Calculate(myTargetInfo, conf, threadIndex);
	}

}

/*
 * Calculate()
 *
 * This function does the actual calculation.
 */

void tpot::Calculate(shared_ptr<info> myTargetInfo, shared_ptr<const plugin_configuration> conf, unsigned short threadIndex)
{

	shared_ptr<fetcher> theFetcher = dynamic_pointer_cast <fetcher> (plugin_factory::Instance()->Plugin("fetcher"));

	// Required source parameters

	param TParam ("T-K");
	param PParam ("P-PA");

	unique_ptr<logger> myThreadedLogger = std::unique_ptr<logger> (logger_factory::Instance()->GetLog("tpotThread #" + boost::lexical_cast<string> (threadIndex)));

	ResetNonLeadingDimension(myTargetInfo);

	myTargetInfo->FirstParam();

	while (AdjustNonLeadingDimension(myTargetInfo))
	{

		myThreadedLogger->Debug("Calculating time " + myTargetInfo->Time().ValidDateTime()->String("%Y%m%d%H") +
								" level " + boost::lexical_cast<string> (myTargetInfo->Level().Value()));

		double PScale = 1;
		double TBase = 0;

		// Source infos
		shared_ptr<info> TInfo;
		shared_ptr<info> PInfo;

		shared_ptr<NFmiGrid> PGrid;

		bool isPressureLevel = (myTargetInfo->Level().Type() == kPressure);

		try
		{

			TInfo = theFetcher->Fetch(conf,
											 myTargetInfo->Time(),
											 myTargetInfo->Level(),
											 TParam);

			if (!isPressureLevel)
			{
				// Source info for P
				PInfo = theFetcher->Fetch(conf,
										  myTargetInfo->Time(),
										  myTargetInfo->Level(),
										  PParam);

				if (PInfo->Param().Unit() == kPa)
				{
					PScale = 0.01;
				}

				PGrid = shared_ptr<NFmiGrid> (PInfo->Grid()->ToNewbaseGrid());
			}
		}
		catch (HPExceptionType e)
		{
			switch (e)
			{
				case kFileDataNotFound:
					itsLogger->Info("Skipping step " + boost::lexical_cast<string> (myTargetInfo->Time().Step()) + ", level " + boost::lexical_cast<string> (myTargetInfo->Level().Value()));
					myTargetInfo->Data()->Fill(kFloatMissing); // Fill data with missing value

					if (conf->StatisticsEnabled())
					{
						conf->Statistics()->AddToMissingCount(myTargetInfo->Grid()->Size());
						conf->Statistics()->AddToValueCount(myTargetInfo->Grid()->Size());
					}

					continue;
					break;

				default:
					throw runtime_error(ClassName() + ": Unable to proceed");
					break;
			}
		}

		unique_ptr<timer> processTimer = unique_ptr<timer> (timer_factory::Instance()->GetTimer());

		if (conf->StatisticsEnabled())
		{
			processTimer->Start();
		}

		if (TInfo->Param().Unit() == kC)
		{
			TBase = 273.15;
		}

		shared_ptr<NFmiGrid> targetGrid(myTargetInfo->Grid()->ToNewbaseGrid());
		shared_ptr<NFmiGrid> TGrid(TInfo->Grid()->ToNewbaseGrid());

		int missingCount = 0;
		int count = 0;

		assert(targetGrid->Size() == myTargetInfo->Data()->Size());

		bool equalGrids = (*myTargetInfo->Grid() == *TInfo->Grid() && (isPressureLevel || *myTargetInfo->Grid() == *PInfo->Grid()));

		string deviceType;

		if (itsUseCuda && equalGrids && threadIndex <= itsCudaDeviceCount)
		{
			deviceType = "GPU";

			size_t N = TGrid->Size();

			double* TPOut;

#ifdef HAVE_CUDA
			cudaMallocHost(reinterpret_cast<void**> (&TPOut), N*sizeof(double));
#else
			TPOut = new double[N];
#endif

			if (!isPressureLevel)
			{
				tpot_cuda::DoCuda(TInfo->Data()->Values(), TBase, PInfo->Data()->Values(), PScale, TPOut, N, 0, threadIndex-1);
			}
			else
			{
				tpot_cuda::DoCuda(TInfo->Data()->Values(), TBase, 0, 0, TPOut, N, myTargetInfo->Level().Value(), threadIndex-1);
			}

			myTargetInfo->Data()->Set(TPOut, N);

#ifdef HAVE_CUDA
			cudaFreeHost(TPOut);
#else
			delete [] TPOut;
#endif
		}
		else
		{

			deviceType = "CPU";

			myTargetInfo->ResetLocation();

			targetGrid->Reset();

			while (myTargetInfo->NextLocation() && targetGrid->Next())
			{
				count++;

				double T = kFloatMissing;
				double P = kFloatMissing;

				InterpolateToPoint(targetGrid, TGrid, equalGrids, T);

				if (isPressureLevel)
				{
					P = myTargetInfo->Level().Value();
				}
				else
				{
					InterpolateToPoint(targetGrid, PGrid, equalGrids, P);
				}

				if (T == kFloatMissing || P == kFloatMissing)
				{
					missingCount++;

					myTargetInfo->Value(kFloatMissing);
					continue;
				}

				double Tp = (T + TBase) * pow((1000 / (P * PScale)), 0.286);

				if (!myTargetInfo->Value(Tp))
				{
					throw runtime_error(ClassName() + ": Failed to set value to matrix");
				}
			}
		}

		if (conf->StatisticsEnabled())
		{
			processTimer->Stop();
			conf->Statistics()->AddToProcessingTime(processTimer->GetTime());

#ifdef DEBUG
			itsLogger->Debug("Calculation took " + boost::lexical_cast<string> (processTimer->GetTime()) + " microseconds on "  + deviceType);
#endif

			conf->Statistics()->AddToMissingCount(missingCount);
			conf->Statistics()->AddToValueCount(count);

		}

		/*
		 * Now we are done for this level
		 *
		 * Clone info-instance to writer since it might change our descriptor places
		 * */

		myThreadedLogger->Info("Missing values: " + boost::lexical_cast<string> (missingCount) + "/" + boost::lexical_cast<string> (count));

		if (conf->FileWriteOption() == kNeons || conf->FileWriteOption() == kMultipleFiles)
		{
			shared_ptr<writer> theWriter = dynamic_pointer_cast <writer> (plugin_factory::Instance()->Plugin("writer"));

			theWriter->ToFile(shared_ptr<info> (new info(*myTargetInfo)), conf);
		}
	}
}
<|endoftext|>


<commit_before>/* -*- Mode: C++; indent-tabs-mode: nil -*- */
#include <schwa/io/logging.h>

#include <cassert>
#include <cstring>
#include <ctime>
#include <iostream>
#include <sstream>
#include <thread>

#include <sys/time.h>

namespace schwa {
namespace io {

const char *
loglevel_name(const LogLevel level) {
  switch (level) {
  case LogLevel::CRITICAL: return LogLevelTraits<LogLevel::CRITICAL>::name; break;
  case LogLevel::ERROR: return LogLevelTraits<LogLevel::ERROR>::name; break;
  case LogLevel::WARNING: return LogLevelTraits<LogLevel::WARNING>::name; break;
  case LogLevel::INFO: return LogLevelTraits<LogLevel::INFO>::name; break;
  case LogLevel::DEBUG: return LogLevelTraits<LogLevel::DEBUG>::name; break;
  default: return nullptr; break;
  }
}

unsigned int
loglevel_value(const LogLevel level) {
  switch (level) {
  case LogLevel::CRITICAL: return LogLevelTraits<LogLevel::CRITICAL>::value; break;
  case LogLevel::ERROR: return LogLevelTraits<LogLevel::ERROR>::value; break;
  case LogLevel::WARNING: return LogLevelTraits<LogLevel::WARNING>::value; break;
  case LogLevel::INFO: return LogLevelTraits<LogLevel::INFO>::value; break;
  case LogLevel::DEBUG: return LogLevelTraits<LogLevel::DEBUG>::value; break;
  default: return 0; break;
  }
}

std::ostream &
operator <<(std::ostream &out, const LogLevel level) {
  return out << loglevel_name(level);
}

size_t
pretty_log_header(const LogLevel level, const char *file, const unsigned int linenum, char *const buf, const size_t buf_len) {
  char now_buf[20];
  time_t now;
  timeval now_tv;
  gettimeofday(&now_tv, nullptr);
  now = now_tv.tv_sec;

  std::strftime(now_buf, sizeof(now_buf), "%FT%T", std::localtime(&now));  // YYYY-MM-DDTHH:mm:ss (19)
  const size_t len = std::strlen(file);
  if (len > 34) {
    file += len - 34;
  }
  return std::snprintf(buf, buf_len, "[%19s.%06zd %-8s %34s:%-5u] ", now_buf, now_tv.tv_usec, loglevel_name(level), file, linenum);
}

size_t
pretty_log_header_threaded(const LogLevel level, const char *file, const unsigned int linenum, char *const buf, const size_t buf_len) {
  char now_buf[20];
  time_t now;
  timeval now_tv;
  gettimeofday(&now_tv, nullptr);
  now = now_tv.tv_sec;

  std::strftime(now_buf, sizeof(now_buf), "%FT%T", std::localtime(&now));  // YYYY-MM-DDTHH:mm:ss (19)

  std::ostringstream thread_id_ss;
  thread_id_ss << std::this_thread::get_id();
  std::string thread_id = thread_id_ss.str();
  if (thread_id.size() > 9) {
    thread_id.resize(9);
  }

  const size_t len = std::strlen(file);
  if (len > 24) {
    file += len - 24;
  }
  return std::snprintf(buf, buf_len, "[%19s.%06zd %-8s %-9s %24s:%-5u] ", now_buf, now_tv.tv_usec, loglevel_name(level), thread_id.c_str(), file, linenum);
}

BasicLogger::Streambuf::Streambuf(std::ostream &ostream, LogLevel threshold) : _ostream(ostream), _threshold(threshold), _level(LogLevel::INFO) { }

BasicLogger::Streambuf::~Streambuf(void) { }

int
BasicLogger::Streambuf::sync(void) {
  if (_level >= _threshold) {
    _ostream << str();
  }
  str("");
  return 0;
}

BasicLogger::BasicLogger(std::ostream &out, LogLevel threshold) : Logger(&_streambuf), _streambuf(out, threshold) { }

BasicLogger::~BasicLogger(void) { }

Logger &
BasicLogger::operator ()(const LogLevel level, const char *const file, const unsigned int linenum) {
  (void)file;
  (void)linenum;
  _streambuf.level(level);
  *this << "[" << level << "]";
  return *this;
}

PrettyLogger::PrettyLogger(std::ostream &out, LogLevel threshold) : BasicLogger(out, threshold) { }

PrettyLogger::~PrettyLogger(void) { }

Logger &
PrettyLogger::operator ()(const LogLevel level, const char *const file, const unsigned int linenum) {
  _streambuf.level(level);
  const size_t used = pretty_log_header(level, file, linenum, _buf, sizeof(_buf));
  assert(used == sizeof(_buf) - 1);
  *this << _buf;
  return *this;
}

ThreadsafeBasicLogger::Streambuf::Streambuf(std::ostream &ostream, LogLevel threshold) : BasicLogger::Streambuf(ostream, threshold) { }

ThreadsafeBasicLogger::Streambuf::~Streambuf(void) { }

int
ThreadsafeBasicLogger::Streambuf::sync(void) {
  const int ret = BasicLogger::Streambuf::sync();
  _lock.unlock();
  return ret;
}

ThreadsafeBasicLogger::ThreadsafeBasicLogger(std::ostream &out, LogLevel threshold) : Logger(&_streambuf), _streambuf(out, threshold) { }

ThreadsafeBasicLogger::~ThreadsafeBasicLogger(void) { }

Logger &
ThreadsafeBasicLogger::operator ()(const LogLevel level, const char *const file, const unsigned int linenum) {
  (void)file;
  (void)linenum;
  _streambuf.lock();
  _streambuf.level(level);
  *this << "[" << level << "]";
  return *this;
}

ThreadsafePrettyLogger::ThreadsafePrettyLogger(std::ostream &out, LogLevel threshold) : ThreadsafeBasicLogger(out, threshold) { }

ThreadsafePrettyLogger::~ThreadsafePrettyLogger(void) { }

Logger &
ThreadsafePrettyLogger::operator ()(const LogLevel level, const char *const file, const unsigned int linenum) {
  _streambuf.lock();
  _streambuf.level(level);
  const size_t used = pretty_log_header_threaded(level, file, linenum, _buf, sizeof(_buf));
  assert(used == sizeof(_buf) - 1);
  *this << _buf;
  return *this;
}

static BasicLogger _default_default_logger(std::clog);
Logger *default_logger = &_default_default_logger;

}  // namespace io
}  // namespace schwa
<commit_msg>Pretty logging format tweaks.<commit_after>/* -*- Mode: C++; indent-tabs-mode: nil -*- */
#include <schwa/io/logging.h>

#include <cassert>
#include <cstring>
#include <ctime>
#include <iostream>
#include <sstream>
#include <thread>

#include <sys/time.h>

namespace schwa {
namespace io {

const char *
loglevel_name(const LogLevel level) {
  switch (level) {
  case LogLevel::CRITICAL: return LogLevelTraits<LogLevel::CRITICAL>::name; break;
  case LogLevel::ERROR: return LogLevelTraits<LogLevel::ERROR>::name; break;
  case LogLevel::WARNING: return LogLevelTraits<LogLevel::WARNING>::name; break;
  case LogLevel::INFO: return LogLevelTraits<LogLevel::INFO>::name; break;
  case LogLevel::DEBUG: return LogLevelTraits<LogLevel::DEBUG>::name; break;
  default: return nullptr; break;
  }
}

unsigned int
loglevel_value(const LogLevel level) {
  switch (level) {
  case LogLevel::CRITICAL: return LogLevelTraits<LogLevel::CRITICAL>::value; break;
  case LogLevel::ERROR: return LogLevelTraits<LogLevel::ERROR>::value; break;
  case LogLevel::WARNING: return LogLevelTraits<LogLevel::WARNING>::value; break;
  case LogLevel::INFO: return LogLevelTraits<LogLevel::INFO>::value; break;
  case LogLevel::DEBUG: return LogLevelTraits<LogLevel::DEBUG>::value; break;
  default: return 0; break;
  }
}

std::ostream &
operator <<(std::ostream &out, const LogLevel level) {
  return out << loglevel_name(level);
}

size_t
pretty_log_header(const LogLevel level, const char *file, const unsigned int linenum, char *const buf, const size_t buf_len) {
  char now_buf[20];
  time_t now;
  timeval now_tv;
  gettimeofday(&now_tv, nullptr);
  now = now_tv.tv_sec;

  std::strftime(now_buf, sizeof(now_buf), "%FT%T", std::localtime(&now));  // YYYY-MM-DDTHH:mm:ss (19)
  const size_t len = std::strlen(file);
  if (len > 35) {
    file += len - 35;
  }
  return std::snprintf(buf, buf_len, "[%19s.%06zd %-8s %35s:%-4u] ", now_buf, now_tv.tv_usec, loglevel_name(level), file, linenum);
}

size_t
pretty_log_header_threaded(const LogLevel level, const char *file, const unsigned int linenum, char *const buf, const size_t buf_len) {
  char now_buf[20];
  time_t now;
  timeval now_tv;
  gettimeofday(&now_tv, nullptr);
  now = now_tv.tv_sec;

  std::strftime(now_buf, sizeof(now_buf), "%FT%T", std::localtime(&now));  // YYYY-MM-DDTHH:mm:ss (19)

  std::ostringstream thread_id_ss;
  thread_id_ss << std::this_thread::get_id();
  std::string thread_id = thread_id_ss.str();
  if (thread_id.size() > 8) {
    thread_id.resize(8);
  }

  const size_t len = std::strlen(file);
  if (len > 25) {
    file += len - 25;
  }
  return std::snprintf(buf, buf_len, "[%19s.%06zd %-8s %25s:%-4u][%-8s] ", now_buf, now_tv.tv_usec, loglevel_name(level), file, linenum, thread_id.c_str());
}

BasicLogger::Streambuf::Streambuf(std::ostream &ostream, LogLevel threshold) : _ostream(ostream), _threshold(threshold), _level(LogLevel::INFO) { }

BasicLogger::Streambuf::~Streambuf(void) { }

int
BasicLogger::Streambuf::sync(void) {
  if (_level >= _threshold) {
    _ostream << str();
  }
  str("");
  return 0;
}

BasicLogger::BasicLogger(std::ostream &out, LogLevel threshold) : Logger(&_streambuf), _streambuf(out, threshold) { }

BasicLogger::~BasicLogger(void) { }

Logger &
BasicLogger::operator ()(const LogLevel level, const char *const file, const unsigned int linenum) {
  (void)file;
  (void)linenum;
  _streambuf.level(level);
  *this << "[" << level << "]";
  return *this;
}

PrettyLogger::PrettyLogger(std::ostream &out, LogLevel threshold) : BasicLogger(out, threshold) { }

PrettyLogger::~PrettyLogger(void) { }

Logger &
PrettyLogger::operator ()(const LogLevel level, const char *const file, const unsigned int linenum) {
  _streambuf.level(level);
  const size_t used = pretty_log_header(level, file, linenum, _buf, sizeof(_buf));
  assert(used == sizeof(_buf) - 1);
  *this << _buf;
  return *this;
}

ThreadsafeBasicLogger::Streambuf::Streambuf(std::ostream &ostream, LogLevel threshold) : BasicLogger::Streambuf(ostream, threshold) { }

ThreadsafeBasicLogger::Streambuf::~Streambuf(void) { }

int
ThreadsafeBasicLogger::Streambuf::sync(void) {
  const int ret = BasicLogger::Streambuf::sync();
  _lock.unlock();
  return ret;
}

ThreadsafeBasicLogger::ThreadsafeBasicLogger(std::ostream &out, LogLevel threshold) : Logger(&_streambuf), _streambuf(out, threshold) { }

ThreadsafeBasicLogger::~ThreadsafeBasicLogger(void) { }

Logger &
ThreadsafeBasicLogger::operator ()(const LogLevel level, const char *const file, const unsigned int linenum) {
  (void)file;
  (void)linenum;
  _streambuf.lock();
  _streambuf.level(level);
  *this << "[" << level << "]";
  return *this;
}

ThreadsafePrettyLogger::ThreadsafePrettyLogger(std::ostream &out, LogLevel threshold) : ThreadsafeBasicLogger(out, threshold) { }

ThreadsafePrettyLogger::~ThreadsafePrettyLogger(void) { }

Logger &
ThreadsafePrettyLogger::operator ()(const LogLevel level, const char *const file, const unsigned int linenum) {
  _streambuf.lock();
  _streambuf.level(level);
  const size_t used = pretty_log_header_threaded(level, file, linenum, _buf, sizeof(_buf));
  assert(used == sizeof(_buf) - 1);
  *this << _buf;
  return *this;
}

static BasicLogger _default_default_logger(std::clog);
Logger *default_logger = &_default_default_logger;

}  // namespace io
}  // namespace schwa
<|endoftext|>