<commit_before>/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/lib/io/record_writer.h"

#include "tensorflow/core/lib/core/coding.h"
#include "tensorflow/core/lib/hash/crc32c.h"
#include "tensorflow/core/lib/io/compression.h"
#include "tensorflow/core/platform/env.h"

namespace tensorflow {
namespace io {
namespace {
bool IsZlibCompressed(const RecordWriterOptions& options) {
  return options.compression_type == RecordWriterOptions::ZLIB_COMPRESSION;
}

bool IsSnappyCompressed(const RecordWriterOptions& options) {
  return options.compression_type == RecordWriterOptions::SNAPPY_COMPRESSION;
}
}  // namespace

RecordWriterOptions RecordWriterOptions::CreateRecordWriterOptions(
    const string& compression_type) {
  RecordWriterOptions options;
#if defined(IS_SLIM_BUILD)
  if (compression_type != compression::kNone) {
    LOG(ERROR) << "Compression is not supported but compression_type is set."
               << " No compression will be used.";
  }
#else
  if (compression_type == compression::kZlib) {
    options.compression_type = io::RecordWriterOptions::ZLIB_COMPRESSION;
    options.zlib_options = io::ZlibCompressionOptions::DEFAULT();
  } else if (compression_type == compression::kGzip) {
    options.compression_type = io::RecordWriterOptions::ZLIB_COMPRESSION;
    options.zlib_options = io::ZlibCompressionOptions::GZIP();
  } else if (compression_type == compression::kSnappy) {
    options.compression_type = io::RecordWriterOptions::SNAPPY_COMPRESSION;
  } else if (compression_type != compression::kNone) {
    LOG(ERROR) << "Unsupported compression_type:" << compression_type
               << ". No compression will be used.";
  }
#endif
  return options;
}

RecordWriter::RecordWriter(WritableFile* dest,
                           const RecordWriterOptions& options)
    : dest_(dest), options_(options) {
#if defined(IS_SLIM_BUILD)
  if (compression_type != compression::kNone) {
    LOG(FATAL) << "Compression is unsupported on mobile platforms.";
  }
#else
  if (IsZlibCompressed(options)) {
    ZlibOutputBuffer* zlib_output_buffer = new ZlibOutputBuffer(
        dest, options.zlib_options.input_buffer_size,
        options.zlib_options.output_buffer_size, options.zlib_options);
    Status s = zlib_output_buffer->Init();
    if (!s.ok()) {
      LOG(FATAL) << "Failed to initialize Zlib inputbuffer. Error: "
                 << s.ToString();
    }
    dest_ = zlib_output_buffer;
  } else if (IsSnappyCompressed(options)) {
    dest_ =
        new SnappyOutputBuffer(dest, options.snappy_options.input_buffer_size,
                               options.snappy_options.output_buffer_size);
  } else if (options.compression_type == RecordWriterOptions::NONE) {
    // Nothing to do
  } else {
    LOG(FATAL) << "Unspecified compression type :" << options.compression_type;
  }
#endif
}

RecordWriter::~RecordWriter() {
  if (dest_ != nullptr) {
    Status s = Close();
    if (!s.ok()) {
      LOG(ERROR) << "Could not finish writing file: " << s;
    }
  }
}

Status RecordWriter::WriteRecord(StringPiece data) {
  if (dest_ == nullptr) {
    return Status(::tensorflow::error::FAILED_PRECONDITION,
                  "Writer not initialized or previously closed");
  }
  // Format of a single record:
  //  uint64    length
  //  uint32    masked crc of length
  //  byte      data[length]
  //  uint32    masked crc of data
  char header[kHeaderSize];
  char footer[kFooterSize];
  PopulateHeader(header, data.data(), data.size());
  PopulateFooter(footer, data.data(), data.size());
  TF_RETURN_IF_ERROR(dest_->Append(StringPiece(header, sizeof(header))));
  TF_RETURN_IF_ERROR(dest_->Append(data));
  return dest_->Append(StringPiece(footer, sizeof(footer)));
}

#if defined(PLATFORM_GOOGLE)
Status RecordWriter::WriteRecord(const absl::Cord& data) {
  if (dest_ == nullptr) {
    return Status(::tensorflow::error::FAILED_PRECONDITION,
                  "Writer not initialized or previously closed");
  }
  // Format of a single record:
  //  uint64    length
  //  uint32    masked crc of length
  //  byte      data[length]
  //  uint32    masked crc of data
  char header[kHeaderSize];
  char footer[kFooterSize];
  PopulateHeader(header, data);
  PopulateFooter(footer, data);
  TF_RETURN_IF_ERROR(dest_->Append(StringPiece(header, sizeof(header))));
  TF_RETURN_IF_ERROR(dest_->Append(data));
  return dest_->Append(StringPiece(footer, sizeof(footer)));
}
#endif

Status RecordWriter::Close() {
  if (dest_ == nullptr) return Status::OK();
  if (IsZlibCompressed(options_) || IsSnappyCompressed(options_)) {
    Status s = dest_->Close();
    delete dest_;
    dest_ = nullptr;
    return s;
  }
  return Status::OK();
}

Status RecordWriter::Flush() {
  if (dest_ == nullptr) {
    return Status(::tensorflow::error::FAILED_PRECONDITION,
                  "Writer not initialized or previously closed");
  }
  return dest_->Flush();
}

}  // namespace io
}  // namespace tensorflow
<commit_msg>Fix slim record_writer compression type<commit_after>/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/lib/io/record_writer.h"

#include "tensorflow/core/lib/core/coding.h"
#include "tensorflow/core/lib/hash/crc32c.h"
#include "tensorflow/core/lib/io/compression.h"
#include "tensorflow/core/platform/env.h"

namespace tensorflow {
namespace io {
namespace {
bool IsZlibCompressed(const RecordWriterOptions& options) {
  return options.compression_type == RecordWriterOptions::ZLIB_COMPRESSION;
}

bool IsSnappyCompressed(const RecordWriterOptions& options) {
  return options.compression_type == RecordWriterOptions::SNAPPY_COMPRESSION;
}
}  // namespace

RecordWriterOptions RecordWriterOptions::CreateRecordWriterOptions(
    const string& compression_type) {
  RecordWriterOptions options;
#if defined(IS_SLIM_BUILD)
  if (compression_type != compression::kNone) {
    LOG(ERROR) << "Compression is not supported but compression_type is set."
               << " No compression will be used.";
  }
#else
  if (compression_type == compression::kZlib) {
    options.compression_type = io::RecordWriterOptions::ZLIB_COMPRESSION;
    options.zlib_options = io::ZlibCompressionOptions::DEFAULT();
  } else if (compression_type == compression::kGzip) {
    options.compression_type = io::RecordWriterOptions::ZLIB_COMPRESSION;
    options.zlib_options = io::ZlibCompressionOptions::GZIP();
  } else if (compression_type == compression::kSnappy) {
    options.compression_type = io::RecordWriterOptions::SNAPPY_COMPRESSION;
  } else if (compression_type != compression::kNone) {
    LOG(ERROR) << "Unsupported compression_type:" << compression_type
               << ". No compression will be used.";
  }
#endif
  return options;
}

RecordWriter::RecordWriter(WritableFile* dest,
                           const RecordWriterOptions& options)
    : dest_(dest), options_(options) {
#if defined(IS_SLIM_BUILD)
  if (options.compression_type != RecordWriterOptions::NONE) {
    LOG(FATAL) << "Compression is unsupported on mobile platforms.";
  }
#else
  if (IsZlibCompressed(options)) {
    ZlibOutputBuffer* zlib_output_buffer = new ZlibOutputBuffer(
        dest, options.zlib_options.input_buffer_size,
        options.zlib_options.output_buffer_size, options.zlib_options);
    Status s = zlib_output_buffer->Init();
    if (!s.ok()) {
      LOG(FATAL) << "Failed to initialize Zlib inputbuffer. Error: "
                 << s.ToString();
    }
    dest_ = zlib_output_buffer;
  } else if (IsSnappyCompressed(options)) {
    dest_ =
        new SnappyOutputBuffer(dest, options.snappy_options.input_buffer_size,
                               options.snappy_options.output_buffer_size);
  } else if (options.compression_type == RecordWriterOptions::NONE) {
    // Nothing to do
  } else {
    LOG(FATAL) << "Unspecified compression type :" << options.compression_type;
  }
#endif
}

RecordWriter::~RecordWriter() {
  if (dest_ != nullptr) {
    Status s = Close();
    if (!s.ok()) {
      LOG(ERROR) << "Could not finish writing file: " << s;
    }
  }
}

Status RecordWriter::WriteRecord(StringPiece data) {
  if (dest_ == nullptr) {
    return Status(::tensorflow::error::FAILED_PRECONDITION,
                  "Writer not initialized or previously closed");
  }
  // Format of a single record:
  //  uint64    length
  //  uint32    masked crc of length
  //  byte      data[length]
  //  uint32    masked crc of data
  char header[kHeaderSize];
  char footer[kFooterSize];
  PopulateHeader(header, data.data(), data.size());
  PopulateFooter(footer, data.data(), data.size());
  TF_RETURN_IF_ERROR(dest_->Append(StringPiece(header, sizeof(header))));
  TF_RETURN_IF_ERROR(dest_->Append(data));
  return dest_->Append(StringPiece(footer, sizeof(footer)));
}

#if defined(PLATFORM_GOOGLE)
Status RecordWriter::WriteRecord(const absl::Cord& data) {
  if (dest_ == nullptr) {
    return Status(::tensorflow::error::FAILED_PRECONDITION,
                  "Writer not initialized or previously closed");
  }
  // Format of a single record:
  //  uint64    length
  //  uint32    masked crc of length
  //  byte      data[length]
  //  uint32    masked crc of data
  char header[kHeaderSize];
  char footer[kFooterSize];
  PopulateHeader(header, data);
  PopulateFooter(footer, data);
  TF_RETURN_IF_ERROR(dest_->Append(StringPiece(header, sizeof(header))));
  TF_RETURN_IF_ERROR(dest_->Append(data));
  return dest_->Append(StringPiece(footer, sizeof(footer)));
}
#endif

Status RecordWriter::Close() {
  if (dest_ == nullptr) return Status::OK();
  if (IsZlibCompressed(options_) || IsSnappyCompressed(options_)) {
    Status s = dest_->Close();
    delete dest_;
    dest_ = nullptr;
    return s;
  }
  return Status::OK();
}

Status RecordWriter::Flush() {
  if (dest_ == nullptr) {
    return Status(::tensorflow::error::FAILED_PRECONDITION,
                  "Writer not initialized or previously closed");
  }
  return dest_->Flush();
}

}  // namespace io
}  // namespace tensorflow
<|endoftext|>


<commit_before><commit_msg>Flush the event loop before/after Action command.<commit_after><|endoftext|>


<commit_before>/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2010 Artem Pavlenko
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/

//$Id: jpeg_reader.cpp 33 2005-04-04 13:01:03Z dane $

// mapnik
#include <mapnik/image_reader.hpp>
#include <mapnik/color.hpp>

// jpeg
extern "C"
{
#include <jpeglib.h>
}

// mapnik

#include <boost/scoped_array.hpp>
#include <boost/utility.hpp>

#ifdef MAPNIK_DEBUG
#include <iostream>
#endif

namespace mapnik
{
    class JpegReader : public image_reader, boost::noncopyable
    {
    private:
        std::string fileName_;
        unsigned width_;
        unsigned height_;
    public:
        explicit JpegReader(const std::string& fileName);
        ~JpegReader();
        unsigned width() const;
        unsigned height() const;
        void read(unsigned x,unsigned y,image_data_32& image);
    private:
        void init();
    };

    namespace
    {
        image_reader* createJpegReader(const std::string& file)
        {
            return new JpegReader(file);
        }
        const bool registered = register_image_reader("jpeg",createJpegReader);
    }

    JpegReader::JpegReader(const std::string& fileName)
        : fileName_(fileName),
          width_(0),
          height_(0)
    {
        init();
    }

    JpegReader::~JpegReader() {}

    void JpegReader::init()
    {
        FILE *fp = fopen(fileName_.c_str(),"rb");
        if (!fp) throw image_reader_exception("JPEG Reader: cannot open image file " + fileName_);

        struct jpeg_decompress_struct cinfo;
        struct jpeg_error_mgr jerr;

        cinfo.err = jpeg_std_error(&jerr);

        jpeg_create_decompress(&cinfo);
        jpeg_stdio_src(&cinfo, fp);
        jpeg_read_header(&cinfo, TRUE);

        jpeg_start_decompress(&cinfo);
        width_ = cinfo.output_width;
        height_ = cinfo.output_height;
        //jpeg_finish_decompress(&cinfo);
        jpeg_destroy_decompress(&cinfo);
        fclose(fp);
    }

    unsigned JpegReader::width() const
    {
        return width_;
    }

    unsigned JpegReader::height() const
    {
        return height_;
    }

    void JpegReader::read(unsigned x0, unsigned y0,image_data_32& image)
    {
        struct jpeg_decompress_struct cinfo;

        FILE *fp = fopen(fileName_.c_str(),"rb");
        if (!fp) throw image_reader_exception("JPEG Reader: cannot open image file " + fileName_);

        struct jpeg_error_mgr jerr;
        cinfo.err = jpeg_std_error(&jerr);

        jpeg_create_decompress(&cinfo);
        jpeg_stdio_src(&cinfo, fp);

        jpeg_read_header(&cinfo, TRUE);
        if (cinfo.out_color_space == JCS_UNKNOWN)
            throw image_reader_exception("JPEG Reader: failed to read unknown color space in " + fileName_);

        jpeg_start_decompress(&cinfo);

        if (cinfo.output_width == 0) {
          jpeg_destroy_decompress (&cinfo);
          fclose(fp);
          throw image_reader_exception("JPEG Reader: failed to read image size of " + fileName_);
        }

        JSAMPARRAY buffer;
        int row_stride;
        unsigned char a,r,g,b;
        row_stride = cinfo.output_width * cinfo.output_components;
        buffer = (*cinfo.mem->alloc_sarray) ((j_common_ptr) &cinfo, JPOOL_IMAGE, row_stride, 1);

        unsigned w = std::min(unsigned(image.width()),width_);
        unsigned h = std::min(unsigned(image.height()),height_);

        unsigned int out_row[w];
        for (unsigned i=0;i<h;++i)
        {
            jpeg_read_scanlines(&cinfo, buffer, 1);
            if (i>=y0 && i<h)
            {
                for (unsigned int x=0; x<w; x++)
                {
                    a = 255; // alpha not supported in jpg
                    r = buffer[0][cinfo.output_components * x];
                    if (cinfo.output_components > 2)
                    {
                        g = buffer[0][cinfo.output_components*x+1];
                        b = buffer[0][cinfo.output_components*x+2];
                    } else {
                        g = r;
                        b = r;
                    }
                    out_row[x] = color(r, g, b, a).rgba();
                }
                image.setRow(i-y0, out_row, w);
            }
        }
        //jpeg_finish_decompress(&cinfo);
        jpeg_destroy_decompress(&cinfo);
        fclose(fp);
    }
}
<commit_msg>add header needed for fopen on linux in jpeg_reader.cpp<commit_after>/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2010 Artem Pavlenko
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/

//$Id: jpeg_reader.cpp 33 2005-04-04 13:01:03Z dane $

// mapnik
#include <mapnik/image_reader.hpp>
#include <mapnik/color.hpp>

// jpeg
extern "C"
{
#include <jpeglib.h>
}

// boost
#include <boost/scoped_array.hpp>
#include <boost/utility.hpp>

// stl
#include <stdio.h>

#ifdef MAPNIK_DEBUG
#include <iostream>
#endif

namespace mapnik
{
    class JpegReader : public image_reader, boost::noncopyable
    {
    private:
        std::string fileName_;
        unsigned width_;
        unsigned height_;
    public:
        explicit JpegReader(const std::string& fileName);
        ~JpegReader();
        unsigned width() const;
        unsigned height() const;
        void read(unsigned x,unsigned y,image_data_32& image);
    private:
        void init();
    };

    namespace
    {
        image_reader* createJpegReader(const std::string& file)
        {
            return new JpegReader(file);
        }
        const bool registered = register_image_reader("jpeg",createJpegReader);
    }

    JpegReader::JpegReader(const std::string& fileName)
        : fileName_(fileName),
          width_(0),
          height_(0)
    {
        init();
    }

    JpegReader::~JpegReader() {}

    void JpegReader::init()
    {
        FILE *fp = fopen(fileName_.c_str(),"rb");
        if (!fp) throw image_reader_exception("JPEG Reader: cannot open image file " + fileName_);

        struct jpeg_decompress_struct cinfo;
        struct jpeg_error_mgr jerr;

        cinfo.err = jpeg_std_error(&jerr);

        jpeg_create_decompress(&cinfo);
        jpeg_stdio_src(&cinfo, fp);
        jpeg_read_header(&cinfo, TRUE);

        jpeg_start_decompress(&cinfo);
        width_ = cinfo.output_width;
        height_ = cinfo.output_height;
        //jpeg_finish_decompress(&cinfo);
        jpeg_destroy_decompress(&cinfo);
        fclose(fp);
    }

    unsigned JpegReader::width() const
    {
        return width_;
    }

    unsigned JpegReader::height() const
    {
        return height_;
    }

    void JpegReader::read(unsigned x0, unsigned y0,image_data_32& image)
    {
        struct jpeg_decompress_struct cinfo;

        FILE *fp = fopen(fileName_.c_str(),"rb");
        if (!fp) throw image_reader_exception("JPEG Reader: cannot open image file " + fileName_);

        struct jpeg_error_mgr jerr;
        cinfo.err = jpeg_std_error(&jerr);

        jpeg_create_decompress(&cinfo);
        jpeg_stdio_src(&cinfo, fp);

        jpeg_read_header(&cinfo, TRUE);
        if (cinfo.out_color_space == JCS_UNKNOWN)
            throw image_reader_exception("JPEG Reader: failed to read unknown color space in " + fileName_);

        jpeg_start_decompress(&cinfo);

        if (cinfo.output_width == 0) {
          jpeg_destroy_decompress (&cinfo);
          fclose(fp);
          throw image_reader_exception("JPEG Reader: failed to read image size of " + fileName_);
        }

        JSAMPARRAY buffer;
        int row_stride;
        unsigned char a,r,g,b;
        row_stride = cinfo.output_width * cinfo.output_components;
        buffer = (*cinfo.mem->alloc_sarray) ((j_common_ptr) &cinfo, JPOOL_IMAGE, row_stride, 1);

        unsigned w = std::min(unsigned(image.width()),width_);
        unsigned h = std::min(unsigned(image.height()),height_);

        unsigned int out_row[w];
        for (unsigned i=0;i<h;++i)
        {
            jpeg_read_scanlines(&cinfo, buffer, 1);
            if (i>=y0 && i<h)
            {
                for (unsigned int x=0; x<w; x++)
                {
                    a = 255; // alpha not supported in jpg
                    r = buffer[0][cinfo.output_components * x];
                    if (cinfo.output_components > 2)
                    {
                        g = buffer[0][cinfo.output_components*x+1];
                        b = buffer[0][cinfo.output_components*x+2];
                    } else {
                        g = r;
                        b = r;
                    }
                    out_row[x] = color(r, g, b, a).rgba();
                }
                image.setRow(i-y0, out_row, w);
            }
        }
        //jpeg_finish_decompress(&cinfo);
        jpeg_destroy_decompress(&cinfo);
        fclose(fp);
    }
}
<|endoftext|>


<commit_before>/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2011 Artem Pavlenko
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/

// mapnik
#include <mapnik/image_reader.hpp>
#include <mapnik/color.hpp>

// jpeg
extern "C"
{
#include <jpeglib.h>
}

// boost
#include <boost/iostreams/device/file.hpp>
#include <boost/iostreams/device/array.hpp>
#include <boost/iostreams/stream.hpp>

// std
#include <cstdio>
#include <memory>

namespace mapnik
{

template <typename T>
class jpeg_reader : public image_reader
{
public:
    typedef T source_type;
    typedef boost::iostreams::stream<source_type> input_stream;
    const static unsigned BUF_SIZE = 4096;
private:
    struct jpeg_stream_wrapper
    {
        jpeg_source_mgr manager;
        input_stream * stream;
        JOCTET buffer[BUF_SIZE];
    };

    struct jpeg_info_guard
    {
        jpeg_info_guard(jpeg_decompress_struct * cinfo)
            : i_(cinfo) {}

        ~jpeg_info_guard()
        {
            jpeg_destroy_decompress(i_);
        }
        jpeg_decompress_struct * i_;
    };

private:
    source_type source_;
    input_stream stream_;
    unsigned width_;
    unsigned height_;
public:
    explicit jpeg_reader(std::string const& file_name);
    explicit jpeg_reader(char const* data, size_t size);
    ~jpeg_reader();
    unsigned width() const;
    unsigned height() const;
    inline bool premultiplied_alpha() const { return true; }
    void read(unsigned x,unsigned y,image_data_32& image);
private:
    void init();
    static void on_error(j_common_ptr cinfo);
    static void on_error_message(j_common_ptr cinfo);
    static void init_source(j_decompress_ptr cinfo);
    static boolean fill_input_buffer(j_decompress_ptr cinfo);
    static void skip(j_decompress_ptr cinfo, long count);
    static void term(j_decompress_ptr cinfo);
    static void attach_stream(j_decompress_ptr cinfo, input_stream* in);
};

namespace
{
image_reader* create_jpeg_reader(std::string const& file)
{
    return new jpeg_reader<boost::iostreams::file_source>(file);
}

image_reader* create_jpeg_reader2(char const* data, size_t size)
{
    return new jpeg_reader<boost::iostreams::array_source>(data, size);
}

const bool registered  = register_image_reader("jpeg",create_jpeg_reader);
const bool registered2 = register_image_reader("jpeg",create_jpeg_reader2);
}

// ctors
template <typename T>
jpeg_reader<T>::jpeg_reader(std::string const& file_name)
    : source_(file_name,std::ios_base::in | std::ios_base::binary),
      stream_(source_),
      width_(0),
      height_(0)
{
    if (!stream_) throw image_reader_exception("cannot open image file "+ file_name);
    init();
}

template <typename T>
jpeg_reader<T>::jpeg_reader(char const* data, size_t size)
    : source_(data, size),
      stream_(source_),
      width_(0),
      height_(0)
{
    if (!stream_) throw image_reader_exception("cannot open image stream");
    init();
}

// dtor
template <typename T>
jpeg_reader<T>::~jpeg_reader() {}

// jpeg stream wrapper
template <typename T>
void jpeg_reader<T>::init_source (j_decompress_ptr cinfo)
{
    jpeg_stream_wrapper* wrap = reinterpret_cast<jpeg_stream_wrapper*>(cinfo->src);
    wrap->stream->seekg(0,std::ios_base::beg);
}

template <typename T>
boolean jpeg_reader<T>::fill_input_buffer (j_decompress_ptr cinfo)
{
    jpeg_stream_wrapper* wrap = reinterpret_cast<jpeg_stream_wrapper*>(cinfo->src);
    wrap->stream->read(reinterpret_cast<char*>(&wrap->buffer[0]),BUF_SIZE);
    std::streamsize size = wrap->stream->gcount();
    wrap->manager.next_input_byte = wrap->buffer;
    wrap->manager.bytes_in_buffer = BUF_SIZE;
    return (size > 0) ? TRUE : FALSE;
}

template <typename T>
void jpeg_reader<T>::skip(j_decompress_ptr cinfo, long count)
{
    if (count <= 0) return; //A zero or negative skip count should be treated as a no-op.
    jpeg_stream_wrapper* wrap = reinterpret_cast<jpeg_stream_wrapper*>(cinfo->src);

    if (wrap->manager.bytes_in_buffer > 0 && count < static_cast<long>(wrap->manager.bytes_in_buffer))
    {
        wrap->manager.bytes_in_buffer -= count;
        wrap->manager.next_input_byte = &wrap->buffer[BUF_SIZE - wrap->manager.bytes_in_buffer];
    }
    else
    {
        wrap->stream->seekg(count, std::ios_base::cur);
        // trigger buffer fill
        wrap->manager.next_input_byte = 0;
        wrap->manager.bytes_in_buffer = 0; //bytes_in_buffer may be zero on return.
    }
}

template <typename T>
void jpeg_reader<T>::term (j_decompress_ptr /*cinfo*/)
{
// no-op
}

template <typename T>
void jpeg_reader<T>::attach_stream (j_decompress_ptr cinfo, input_stream* in)
{
    if (cinfo->src == 0)
    {
        cinfo->src = (struct jpeg_source_mgr *)
            (*cinfo->mem->alloc_small) ((j_common_ptr) cinfo, JPOOL_PERMANENT, sizeof(jpeg_stream_wrapper));
    }
    jpeg_reader::jpeg_stream_wrapper * src = reinterpret_cast<jpeg_reader::jpeg_stream_wrapper*> (cinfo->src);
    src->manager.init_source = init_source;
    src->manager.fill_input_buffer = fill_input_buffer;
    src->manager.skip_input_data = skip;
    src->manager.resync_to_restart = jpeg_resync_to_restart;
    src->manager.term_source = term;
    src->manager.bytes_in_buffer = 0;
    src->manager.next_input_byte = 0;
    src->stream = in;
}

template <typename T>
void jpeg_reader<T>::on_error(j_common_ptr /*cinfo*/)
{
}

template <typename T>
void jpeg_reader<T>::on_error_message(j_common_ptr cinfo)
{
    char buffer[JMSG_LENGTH_MAX];
    (*cinfo->err->format_message)(cinfo, buffer);
    throw image_reader_exception(std::string("JPEG Reader: libjpeg could not read image: ") + buffer);
}

template <typename T>
void jpeg_reader<T>::init()
{
    jpeg_decompress_struct cinfo;
    jpeg_info_guard iguard(&cinfo);
    jpeg_error_mgr jerr;
    cinfo.err = jpeg_std_error(&jerr);
    jerr.error_exit = on_error;
    jerr.output_message = on_error_message;
    jpeg_create_decompress(&cinfo);
    attach_stream(&cinfo, &stream_);
    int ret = jpeg_read_header(&cinfo, TRUE);
    if (ret != JPEG_HEADER_OK)
        throw image_reader_exception("JPEG Reader: failed to read header");
    jpeg_start_decompress(&cinfo);
    width_ = cinfo.output_width;
    height_ = cinfo.output_height;

    if (cinfo.out_color_space == JCS_UNKNOWN)
    {
        throw image_reader_exception("JPEG Reader: failed to read unknown color space");
    }
    if (cinfo.output_width == 0 || cinfo.output_height == 0)
    {
        throw image_reader_exception("JPEG Reader: failed to read image size of");
    }
}

template <typename T>
unsigned jpeg_reader<T>::width() const
{
    return width_;
}

template <typename T>
unsigned jpeg_reader<T>::height() const
{
    return height_;
}

template <typename T>
void jpeg_reader<T>::read(unsigned x0, unsigned y0, image_data_32& image)
{
    stream_.clear();
    stream_.seekg(0, std::ios_base::beg);

    jpeg_decompress_struct cinfo;
    jpeg_info_guard iguard(&cinfo);
    jpeg_error_mgr jerr;
    cinfo.err = jpeg_std_error(&jerr);
    jerr.error_exit = on_error;
    jerr.output_message = on_error_message;
    jpeg_create_decompress(&cinfo);
    attach_stream(&cinfo, &stream_);
    int ret = jpeg_read_header(&cinfo, TRUE);
    if (ret != JPEG_HEADER_OK) throw image_reader_exception("JPEG Reader read(): failed to read header");
    jpeg_start_decompress(&cinfo);
    JSAMPARRAY buffer;
    int row_stride;
    unsigned char a,r,g,b;
    row_stride = cinfo.output_width * cinfo.output_components;
    buffer = (*cinfo.mem->alloc_sarray) ((j_common_ptr) &cinfo, JPOOL_IMAGE, row_stride, 1);

    unsigned w = std::min(unsigned(image.width()),width_ - x0);
    unsigned h = std::min(unsigned(image.height()),height_ - y0);

    const std::unique_ptr<unsigned int[]> out_row(new unsigned int[w]);
    unsigned row = 0;
    while (cinfo.output_scanline < cinfo.output_height)
    {
        jpeg_read_scanlines(&cinfo, buffer, 1);
        if (row >= y0 && row < y0 + h)
        {
            for (unsigned int x = 0; x < w; ++x)
            {
                unsigned col = x + x0;
                a = 255; // alpha not supported in jpg
                r = buffer[0][cinfo.output_components * col];
                if (cinfo.output_components > 2)
                {
                    g = buffer[0][cinfo.output_components * col + 1];
                    b = buffer[0][cinfo.output_components * col + 2];
                } else {
                    g = r;
                    b = r;
                }
                out_row[x] = color(r, g, b, a).rgba();
            }
            image.setRow(row - y0, out_row.get(), w);
        }
        ++row;
    }
    jpeg_finish_decompress(&cinfo);
}

}
<commit_msg>fix jpeg reading regression after #1805 - closes #2123 (patch from @clundgren)<commit_after>/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2011 Artem Pavlenko
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/

// mapnik
#include <mapnik/image_reader.hpp>
#include <mapnik/color.hpp>

// jpeg
extern "C"
{
#include <jpeglib.h>
}

// boost
#include <boost/iostreams/device/file.hpp>
#include <boost/iostreams/device/array.hpp>
#include <boost/iostreams/stream.hpp>

// std
#include <cstdio>
#include <memory>

namespace mapnik
{

template <typename T>
class jpeg_reader : public image_reader
{
public:
    typedef T source_type;
    typedef boost::iostreams::stream<source_type> input_stream;
    const static unsigned BUF_SIZE = 4096;
private:
    struct jpeg_stream_wrapper
    {
        jpeg_source_mgr manager;
        input_stream * stream;
        JOCTET buffer[BUF_SIZE];
    };

    struct jpeg_info_guard
    {
        jpeg_info_guard(jpeg_decompress_struct * cinfo)
            : i_(cinfo) {}

        ~jpeg_info_guard()
        {
            jpeg_destroy_decompress(i_);
        }
        jpeg_decompress_struct * i_;
    };

private:
    source_type source_;
    input_stream stream_;
    unsigned width_;
    unsigned height_;
public:
    explicit jpeg_reader(std::string const& file_name);
    explicit jpeg_reader(char const* data, size_t size);
    ~jpeg_reader();
    unsigned width() const;
    unsigned height() const;
    inline bool premultiplied_alpha() const { return true; }
    void read(unsigned x,unsigned y,image_data_32& image);
private:
    void init();
    static void on_error(j_common_ptr cinfo);
    static void on_error_message(j_common_ptr cinfo);
    static void init_source(j_decompress_ptr cinfo);
    static boolean fill_input_buffer(j_decompress_ptr cinfo);
    static void skip(j_decompress_ptr cinfo, long count);
    static void term(j_decompress_ptr cinfo);
    static void attach_stream(j_decompress_ptr cinfo, input_stream* in);
};

namespace
{
image_reader* create_jpeg_reader(std::string const& file)
{
    return new jpeg_reader<boost::iostreams::file_source>(file);
}

image_reader* create_jpeg_reader2(char const* data, size_t size)
{
    return new jpeg_reader<boost::iostreams::array_source>(data, size);
}

const bool registered  = register_image_reader("jpeg",create_jpeg_reader);
const bool registered2 = register_image_reader("jpeg",create_jpeg_reader2);
}

// ctors
template <typename T>
jpeg_reader<T>::jpeg_reader(std::string const& file_name)
    : source_(file_name,std::ios_base::in | std::ios_base::binary),
      stream_(source_),
      width_(0),
      height_(0)
{
    if (!stream_) throw image_reader_exception("cannot open image file "+ file_name);
    init();
}

template <typename T>
jpeg_reader<T>::jpeg_reader(char const* data, size_t size)
    : source_(data, size),
      stream_(source_),
      width_(0),
      height_(0)
{
    if (!stream_) throw image_reader_exception("cannot open image stream");
    init();
}

// dtor
template <typename T>
jpeg_reader<T>::~jpeg_reader() {}

// jpeg stream wrapper
template <typename T>
void jpeg_reader<T>::init_source (j_decompress_ptr cinfo)
{
    jpeg_stream_wrapper* wrap = reinterpret_cast<jpeg_stream_wrapper*>(cinfo->src);
    wrap->stream->seekg(0,std::ios_base::beg);
}

template <typename T>
boolean jpeg_reader<T>::fill_input_buffer (j_decompress_ptr cinfo)
{
    jpeg_stream_wrapper* wrap = reinterpret_cast<jpeg_stream_wrapper*>(cinfo->src);
    wrap->stream->read(reinterpret_cast<char*>(&wrap->buffer[0]),BUF_SIZE);
    std::streamsize size = wrap->stream->gcount();
    wrap->manager.next_input_byte = wrap->buffer;
    wrap->manager.bytes_in_buffer = BUF_SIZE;
    return (size > 0) ? TRUE : FALSE;
}

template <typename T>
void jpeg_reader<T>::skip(j_decompress_ptr cinfo, long count)
{
    if (count <= 0) return; //A zero or negative skip count should be treated as a no-op.
    jpeg_stream_wrapper* wrap = reinterpret_cast<jpeg_stream_wrapper*>(cinfo->src);

    if (wrap->manager.bytes_in_buffer > 0 && count < static_cast<long>(wrap->manager.bytes_in_buffer))
    {
        wrap->manager.bytes_in_buffer -= count;
        wrap->manager.next_input_byte = &wrap->buffer[BUF_SIZE - wrap->manager.bytes_in_buffer];
    }
    else
    {
        wrap->stream->seekg(count - wrap->manager.bytes_in_buffer, std::ios_base::cur);
        // trigger buffer fill
        wrap->manager.next_input_byte = 0;
        wrap->manager.bytes_in_buffer = 0; //bytes_in_buffer may be zero on return.
    }
}

template <typename T>
void jpeg_reader<T>::term (j_decompress_ptr /*cinfo*/)
{
// no-op
}

template <typename T>
void jpeg_reader<T>::attach_stream (j_decompress_ptr cinfo, input_stream* in)
{
    if (cinfo->src == 0)
    {
        cinfo->src = (struct jpeg_source_mgr *)
            (*cinfo->mem->alloc_small) ((j_common_ptr) cinfo, JPOOL_PERMANENT, sizeof(jpeg_stream_wrapper));
    }
    jpeg_reader::jpeg_stream_wrapper * src = reinterpret_cast<jpeg_reader::jpeg_stream_wrapper*> (cinfo->src);
    src->manager.init_source = init_source;
    src->manager.fill_input_buffer = fill_input_buffer;
    src->manager.skip_input_data = skip;
    src->manager.resync_to_restart = jpeg_resync_to_restart;
    src->manager.term_source = term;
    src->manager.bytes_in_buffer = 0;
    src->manager.next_input_byte = 0;
    src->stream = in;
}

template <typename T>
void jpeg_reader<T>::on_error(j_common_ptr /*cinfo*/)
{
}

template <typename T>
void jpeg_reader<T>::on_error_message(j_common_ptr cinfo)
{
    char buffer[JMSG_LENGTH_MAX];
    (*cinfo->err->format_message)(cinfo, buffer);
    throw image_reader_exception(std::string("JPEG Reader: libjpeg could not read image: ") + buffer);
}

template <typename T>
void jpeg_reader<T>::init()
{
    jpeg_decompress_struct cinfo;
    jpeg_info_guard iguard(&cinfo);
    jpeg_error_mgr jerr;
    cinfo.err = jpeg_std_error(&jerr);
    jerr.error_exit = on_error;
    jerr.output_message = on_error_message;
    jpeg_create_decompress(&cinfo);
    attach_stream(&cinfo, &stream_);
    int ret = jpeg_read_header(&cinfo, TRUE);
    if (ret != JPEG_HEADER_OK)
        throw image_reader_exception("JPEG Reader: failed to read header");
    jpeg_start_decompress(&cinfo);
    width_ = cinfo.output_width;
    height_ = cinfo.output_height;

    if (cinfo.out_color_space == JCS_UNKNOWN)
    {
        throw image_reader_exception("JPEG Reader: failed to read unknown color space");
    }
    if (cinfo.output_width == 0 || cinfo.output_height == 0)
    {
        throw image_reader_exception("JPEG Reader: failed to read image size of");
    }
}

template <typename T>
unsigned jpeg_reader<T>::width() const
{
    return width_;
}

template <typename T>
unsigned jpeg_reader<T>::height() const
{
    return height_;
}

template <typename T>
void jpeg_reader<T>::read(unsigned x0, unsigned y0, image_data_32& image)
{
    stream_.clear();
    stream_.seekg(0, std::ios_base::beg);

    jpeg_decompress_struct cinfo;
    jpeg_info_guard iguard(&cinfo);
    jpeg_error_mgr jerr;
    cinfo.err = jpeg_std_error(&jerr);
    jerr.error_exit = on_error;
    jerr.output_message = on_error_message;
    jpeg_create_decompress(&cinfo);
    attach_stream(&cinfo, &stream_);
    int ret = jpeg_read_header(&cinfo, TRUE);
    if (ret != JPEG_HEADER_OK) throw image_reader_exception("JPEG Reader read(): failed to read header");
    jpeg_start_decompress(&cinfo);
    JSAMPARRAY buffer;
    int row_stride;
    unsigned char a,r,g,b;
    row_stride = cinfo.output_width * cinfo.output_components;
    buffer = (*cinfo.mem->alloc_sarray) ((j_common_ptr) &cinfo, JPOOL_IMAGE, row_stride, 1);

    unsigned w = std::min(unsigned(image.width()),width_ - x0);
    unsigned h = std::min(unsigned(image.height()),height_ - y0);

    const std::unique_ptr<unsigned int[]> out_row(new unsigned int[w]);
    unsigned row = 0;
    while (cinfo.output_scanline < cinfo.output_height)
    {
        jpeg_read_scanlines(&cinfo, buffer, 1);
        if (row >= y0 && row < y0 + h)
        {
            for (unsigned int x = 0; x < w; ++x)
            {
                unsigned col = x + x0;
                a = 255; // alpha not supported in jpg
                r = buffer[0][cinfo.output_components * col];
                if (cinfo.output_components > 2)
                {
                    g = buffer[0][cinfo.output_components * col + 1];
                    b = buffer[0][cinfo.output_components * col + 2];
                } else {
                    g = r;
                    b = r;
                }
                out_row[x] = color(r, g, b, a).rgba();
            }
            image.setRow(row - y0, out_row.get(), w);
        }
        ++row;
    }
    jpeg_finish_decompress(&cinfo);
}

}
<|endoftext|>


<commit_before>#include <silicium/aligned_ref.hpp>
#include <silicium/alignment_of.hpp>
#include <silicium/arithmetic/add.hpp>
#include <silicium/arithmetic/overflow_or.hpp>
#include <silicium/array_view.hpp>
#include <silicium/asio/accepting_source.hpp>
#include <silicium/asio/async.hpp>
#include <silicium/asio/async_source.hpp>
#include <silicium/asio/connecting_observable.hpp>
#include <silicium/asio/connecting_source.hpp>
#include <silicium/asio/post_forwarder.hpp>
#include <silicium/asio/posting_observable.hpp>
#include <silicium/asio/process_output.hpp>
#include <silicium/asio/reading_observable.hpp>
#include <silicium/asio/socket_sink.hpp>
#include <silicium/asio/socket_source.hpp>
#include <silicium/asio/tcp_acceptor.hpp>
#include <silicium/asio/timer.hpp>
#include <silicium/asio/use_observable.hpp>
#include <silicium/asio/writing_observable.hpp>
#include <silicium/block_thread.hpp>
#include <silicium/boost_threading.hpp>
#include <silicium/bounded_int.hpp>
#include <silicium/buffer.hpp>
#include <silicium/byte.hpp>
#include <silicium/byte_order_intrinsics.hpp>
#include <silicium/c_string.hpp>
#include <silicium/config.hpp>
#include <silicium/detail/argument_of.hpp>
#include <silicium/detail/basic_dynamic_library.hpp>
#include <silicium/detail/element_from_optional_like.hpp>
#include <silicium/detail/integer_sequence.hpp>
#include <silicium/detail/line_source.hpp>
#include <silicium/detail/proper_value_function.hpp>
#include <silicium/dynamic_library.hpp>
#include <silicium/environment_variables.hpp>
#include <silicium/error_code.hpp>
#include <silicium/error_handler.hpp>
#include <silicium/error_or.hpp>
#include <silicium/exchange.hpp>
#include <silicium/expected.hpp>
#include <silicium/explicit_operator_bool.hpp>
#include <silicium/file_handle.hpp>
#include <silicium/function.hpp>
#include <silicium/future.hpp>
#include <silicium/get_last_error.hpp>
#include <silicium/html/generator.hpp>
#include <silicium/html/tree.hpp>
#include <silicium/http/generate_header.hpp>
#include <silicium/http/generate_request.hpp>
#include <silicium/http/generate_response.hpp>
#include <silicium/http/http.hpp>
#include <silicium/http/parse_request.hpp>
#include <silicium/http/parse_response.hpp>
#include <silicium/http/receive_request.hpp>
#include <silicium/http/request_parser_sink.hpp>
#include <silicium/http/uri.hpp>
#include <silicium/identity.hpp>
#include <silicium/initialize_array.hpp>
#include <silicium/is_handle.hpp>
#include <silicium/iterator_range.hpp>
#include <silicium/lossless_cast.hpp>
#include <silicium/make_array.hpp>
#include <silicium/make_destructor.hpp>
#include <silicium/make_unique.hpp>
#include <silicium/memory_range.hpp>
#include <silicium/move.hpp>
#include <silicium/move_if_noexcept.hpp>
#include <silicium/native_file_descriptor.hpp>
#include <silicium/noexcept_string.hpp>
#include <silicium/null_mutex.hpp>
#include <silicium/observable/bridge.hpp>
#include <silicium/observable/cache.hpp>
#include <silicium/observable/constant.hpp>
#include <silicium/observable/consume.hpp>
#include <silicium/observable/coroutine.hpp>
#include <silicium/observable/coroutine_generator.hpp>
#include <silicium/observable/deref_optional.hpp>
#include <silicium/observable/empty.hpp>
#include <silicium/observable/end.hpp>
#include <silicium/observable/enumerate.hpp>
#include <silicium/observable/erase_shared.hpp>
#include <silicium/observable/erase_unique.hpp>
#include <silicium/observable/erased_observer.hpp>
#include <silicium/observable/error_or_enumerate.hpp>
#include <silicium/observable/extensible_observer.hpp>
#include <silicium/observable/filter.hpp>
#include <silicium/observable/finite_state_machine.hpp>
#include <silicium/observable/flatten.hpp>
#include <silicium/observable/for_each.hpp>
#include <silicium/observable/function.hpp>
#include <silicium/observable/function_observer.hpp>
#include <silicium/observable/generator.hpp>
#include <silicium/observable/limited.hpp>
#include <silicium/observable/observable.hpp>
#include <silicium/observable/observer.hpp>
#include <silicium/observable/on_first.hpp>
#include <silicium/observable/ptr.hpp>
#include <silicium/observable/ready_future.hpp>
#include <silicium/observable/ref.hpp>
#include <silicium/observable/source.hpp>
#include <silicium/observable/spawn_coroutine.hpp>
#include <silicium/observable/spawn_observable.hpp>
#include <silicium/observable/take.hpp>
#include <silicium/observable/thread.hpp>
#include <silicium/observable/thread_generator.hpp>
#include <silicium/observable/total_consumer.hpp>
#include <silicium/observable/transform.hpp>
#include <silicium/observable/transform_if_initialized.hpp>
#include <silicium/observable/tuple.hpp>
#include <silicium/observable/variant.hpp>
#include <silicium/observable/virtualized.hpp>
#include <silicium/observable/while.hpp>
#include <silicium/observable/yield_context.hpp>
#include <silicium/observable2.hpp>
#include <silicium/optional.hpp>
#include <silicium/os_string.hpp>
#include <silicium/pipe.hpp>
#include <silicium/process_handle.hpp>
#include <silicium/program_options.hpp>
#include <silicium/ptr_adaptor.hpp>
#include <silicium/range_value.hpp>
#include <silicium/read_file.hpp>
#include <silicium/serialization.hpp>
#include <silicium/sink/append.hpp>
#include <silicium/sink/buffer.hpp>
#include <silicium/sink/buffering_sink.hpp>
#include <silicium/sink/container_buffer.hpp>
#include <silicium/sink/copy.hpp>
#include <silicium/sink/file_sink.hpp>
#include <silicium/sink/function_sink.hpp>
#include <silicium/sink/iterator_sink.hpp>
#include <silicium/sink/multi_sink.hpp>
#include <silicium/sink/ostream_sink.hpp>
#include <silicium/sink/ptr_sink.hpp>
#include <silicium/sink/sink.hpp>
#include <silicium/sink/throwing_sink.hpp>
#include <silicium/sink/virtualized_sink.hpp>
#include <silicium/source/buffering_source.hpp>
#include <silicium/source/empty.hpp>
#include <silicium/source/enumerating_source.hpp>
#include <silicium/source/error_extracting_source.hpp>
#include <silicium/source/filter_source.hpp>
#include <silicium/source/generator_source.hpp>
#include <silicium/source/memory_source.hpp>
#include <silicium/source/observable_source.hpp>
#include <silicium/source/ptr_source.hpp>
#include <silicium/source/range_source.hpp>
#include <silicium/source/received_from_socket_source.hpp>
#include <silicium/source/single_source.hpp>
#include <silicium/source/source.hpp>
#include <silicium/source/throwing_source.hpp>
#include <silicium/source/transforming_source.hpp>
#include <silicium/source/virtualized_source.hpp>
#include <silicium/std_threading.hpp>
#include <silicium/steady_clock.hpp>
#include <silicium/success.hpp>
#include <silicium/terminate_on_exception.hpp>
#include <silicium/then.hpp>
#include <silicium/throw_last_error.hpp>
#include <silicium/to_shared.hpp>
#include <silicium/to_unique.hpp>
#include <silicium/trait.hpp>
#include <silicium/type_traits.hpp>
#include <silicium/utility.hpp>
#include <silicium/variant.hpp>
#include <silicium/vector.hpp>
#include <silicium/version.hpp>
#include <silicium/win32/dynamic_library_impl.hpp>
#include <silicium/win32/process_handle.hpp>
#include <silicium/win32/win32.hpp>
#include <silicium/write.hpp>
#include <silicium/zlib/deflating_sink.hpp>
#include <silicium/zlib/inflating_sink.hpp>
#include <silicium/zlib/zlib.hpp>
<commit_msg>update Windows #include test<commit_after>#include <silicium/aligned_ref.hpp>
#include <silicium/alignment_of.hpp>
#include <silicium/arithmetic/add.hpp>
#include <silicium/arithmetic/overflow_or.hpp>
#include <silicium/array_view.hpp>
#include <silicium/asio/accepting_source.hpp>
#include <silicium/asio/async.hpp>
#include <silicium/asio/async_source.hpp>
#include <silicium/asio/block_thread.hpp>
#include <silicium/asio/connecting_observable.hpp>
#include <silicium/asio/connecting_source.hpp>
#include <silicium/asio/post_forwarder.hpp>
#include <silicium/asio/posting_observable.hpp>
#include <silicium/asio/process_output.hpp>
#include <silicium/asio/reading_observable.hpp>
#include <silicium/asio/socket_sink.hpp>
#include <silicium/asio/socket_source.hpp>
#include <silicium/asio/tcp_acceptor.hpp>
#include <silicium/asio/timer.hpp>
#include <silicium/asio/use_observable.hpp>
#include <silicium/asio/writing_observable.hpp>
#include <silicium/boost_threading.hpp>
#include <silicium/bounded_int.hpp>
#include <silicium/byte.hpp>
#include <silicium/byte_order_intrinsics.hpp>
#include <silicium/c_string.hpp>
#include <silicium/config.hpp>
#include <silicium/detail/argument_of.hpp>
#include <silicium/detail/basic_dynamic_library.hpp>
#include <silicium/detail/element_from_optional_like.hpp>
#include <silicium/detail/integer_sequence.hpp>
#include <silicium/detail/line_source.hpp>
#include <silicium/detail/proper_value_function.hpp>
#include <silicium/detail/then.hpp>
#include <silicium/dynamic_library.hpp>
#include <silicium/environment_variables.hpp>
#include <silicium/error_code.hpp>
#include <silicium/error_handler.hpp>
#include <silicium/error_or.hpp>
#include <silicium/exchange.hpp>
#include <silicium/expected.hpp>
#include <silicium/explicit_operator_bool.hpp>
#include <silicium/file_handle.hpp>
#include <silicium/function.hpp>
#include <silicium/get_last_error.hpp>
#include <silicium/html/generator.hpp>
#include <silicium/html/tree.hpp>
#include <silicium/http/generate_header.hpp>
#include <silicium/http/generate_request.hpp>
#include <silicium/http/generate_response.hpp>
#include <silicium/http/http.hpp>
#include <silicium/http/parse_request.hpp>
#include <silicium/http/parse_response.hpp>
#include <silicium/http/receive_request.hpp>
#include <silicium/http/request_parser_sink.hpp>
#include <silicium/http/uri.hpp>
#include <silicium/identity.hpp>
#include <silicium/initialize_array.hpp>
#include <silicium/iterator_range.hpp>
#include <silicium/lossless_cast.hpp>
#include <silicium/make_array.hpp>
#include <silicium/make_destructor.hpp>
#include <silicium/make_unique.hpp>
#include <silicium/memory_range.hpp>
#include <silicium/move.hpp>
#include <silicium/move_if_noexcept.hpp>
#include <silicium/native_file_descriptor.hpp>
#include <silicium/noexcept_string.hpp>
#include <silicium/null_mutex.hpp>
#include <silicium/observable/bridge.hpp>
#include <silicium/observable/buffer.hpp>
#include <silicium/observable/cache.hpp>
#include <silicium/observable/constant.hpp>
#include <silicium/observable/consume.hpp>
#include <silicium/observable/coroutine.hpp>
#include <silicium/observable/coroutine_generator.hpp>
#include <silicium/observable/deref_optional.hpp>
#include <silicium/observable/empty.hpp>
#include <silicium/observable/end.hpp>
#include <silicium/observable/enumerate.hpp>
#include <silicium/observable/erase_shared.hpp>
#include <silicium/observable/erase_unique.hpp>
#include <silicium/observable/erased_observer.hpp>
#include <silicium/observable/error_or_enumerate.hpp>
#include <silicium/observable/extensible_observer.hpp>
#include <silicium/observable/filter.hpp>
#include <silicium/observable/finite_state_machine.hpp>
#include <silicium/observable/flatten.hpp>
#include <silicium/observable/for_each.hpp>
#include <silicium/observable/function.hpp>
#include <silicium/observable/function_observer.hpp>
#include <silicium/observable/generator.hpp>
#include <silicium/observable/limited.hpp>
#include <silicium/observable/observable.hpp>
#include <silicium/observable/observer.hpp>
#include <silicium/observable/on_first.hpp>
#include <silicium/observable/ptr.hpp>
#include <silicium/observable/ready_future.hpp>
#include <silicium/observable/ref.hpp>
#include <silicium/observable/source.hpp>
#include <silicium/observable/spawn_coroutine.hpp>
#include <silicium/observable/spawn_observable.hpp>
#include <silicium/observable/take.hpp>
#include <silicium/observable/thread.hpp>
#include <silicium/observable/thread_generator.hpp>
#include <silicium/observable/total_consumer.hpp>
#include <silicium/observable/transform.hpp>
#include <silicium/observable/transform_if_initialized.hpp>
#include <silicium/observable/tuple.hpp>
#include <silicium/observable/variant.hpp>
#include <silicium/observable/virtualized.hpp>
#include <silicium/observable/while.hpp>
#include <silicium/observable/yield_context.hpp>
#include <silicium/optional.hpp>
#include <silicium/os_string.hpp>
#include <silicium/pipe.hpp>
#include <silicium/process_handle.hpp>
#include <silicium/program_options.hpp>
#include <silicium/ptr_adaptor.hpp>
#include <silicium/range_value.hpp>
#include <silicium/read.hpp>
#include <silicium/sink/append.hpp>
#include <silicium/sink/buffer.hpp>
#include <silicium/sink/buffering_sink.hpp>
#include <silicium/sink/container_buffer.hpp>
#include <silicium/sink/copy.hpp>
#include <silicium/sink/file_sink.hpp>
#include <silicium/sink/function_sink.hpp>
#include <silicium/sink/iterator_sink.hpp>
#include <silicium/sink/multi_sink.hpp>
#include <silicium/sink/ostream_sink.hpp>
#include <silicium/sink/ptr_sink.hpp>
#include <silicium/sink/sink.hpp>
#include <silicium/sink/throwing_sink.hpp>
#include <silicium/sink/virtualized_sink.hpp>
#include <silicium/source/buffering_source.hpp>
#include <silicium/source/empty.hpp>
#include <silicium/source/enumerating_source.hpp>
#include <silicium/source/error_extracting_source.hpp>
#include <silicium/source/filter_source.hpp>
#include <silicium/source/generator_source.hpp>
#include <silicium/source/memory_source.hpp>
#include <silicium/source/observable_source.hpp>
#include <silicium/source/ptr_source.hpp>
#include <silicium/source/range_source.hpp>
#include <silicium/source/received_from_socket_source.hpp>
#include <silicium/source/single_source.hpp>
#include <silicium/source/source.hpp>
#include <silicium/source/throwing_source.hpp>
#include <silicium/source/transforming_source.hpp>
#include <silicium/source/virtualized_source.hpp>
#include <silicium/std_threading.hpp>
#include <silicium/steady_clock.hpp>
#include <silicium/success.hpp>
#include <silicium/terminate_on_exception.hpp>
#include <silicium/throw_last_error.hpp>
#include <silicium/to_shared.hpp>
#include <silicium/to_unique.hpp>
#include <silicium/trait.hpp>
#include <silicium/type_traits.hpp>
#include <silicium/variant.hpp>
#include <silicium/version.hpp>
#include <silicium/win32/dynamic_library_impl.hpp>
#include <silicium/win32/process_handle.hpp>
#include <silicium/win32/win32.hpp>
#include <silicium/write.hpp>
#include <silicium/zlib/deflating_sink.hpp>
#include <silicium/zlib/inflating_sink.hpp>
#include <silicium/zlib/zlib.hpp>
<|endoftext|>


<commit_before>/*
 * Distributed under the OSI-approved Apache License, Version 2.0.  See
 * accompanying file Copyright.txt for details.
 */
#include <cstdint>
#include <cstring>

#include <iostream>
#include <stdexcept>

#include <adios2.h>

#include <gtest/gtest.h>

#include "../SmallTestData.h"

std::string engineName; // comes from command line

class AppendTimeStepTest : public ::testing::Test
{
public:
    AppendTimeStepTest() = default;

    SmallTestData m_TestData;
};

//******************************************************************************
// 1D 1x8 test data
//******************************************************************************

// ADIOS2 HDF5 write, then append, then read back
TEST_F(AppendTimeStepTest, ADIOS2HDF5WriteAppendRead)
{
    // Each process would write a 1x8 array and all processes would
    // form a mpiSize * Nx 1D array
    std::string fname = "appendTest.h5";

    int mpiRank = 0, mpiSize = 1;
    // Number of rows
    const std::size_t Nx = 8;

    // Number of steps
    const std::size_t NSteps = 3;

#if ADIOS2_USE_MPI
    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
#endif

#if ADIOS2_USE_MPI
    adios2::ADIOS adios(MPI_COMM_WORLD);
#else
    adios2::ADIOS adios;
#endif

    // Write test data using HDF5 engine
    {
        adios2::IO io = adios.DeclareIO("TestIO");

        // Declare 1D variables (NumOfProcesses * Nx)
        // The local process' part (start, count) can be defined now or later
        // before Write().
        {
            adios2::Dims shape{static_cast<unsigned int>(Nx * mpiSize)};
            adios2::Dims start{static_cast<unsigned int>(Nx * mpiRank)};
            adios2::Dims count{static_cast<unsigned int>(Nx)};

            auto var_iString = io.DefineVariable<std::string>("iString");
            auto var_i8 = io.DefineVariable<int8_t>("i8", shape, start, count);
            auto var_i16 =
                io.DefineVariable<int16_t>("i16", shape, start, count);
            auto var_i32 =
                io.DefineVariable<int32_t>("i32", shape, start, count);
            auto var_i64 =
                io.DefineVariable<int64_t>("i64", shape, start, count);
            auto var_u8 = io.DefineVariable<uint8_t>("u8", shape, start, count);
            auto var_u16 =
                io.DefineVariable<uint16_t>("u16", shape, start, count);
            auto var_u32 =
                io.DefineVariable<uint32_t>("u32", shape, start, count);
            auto var_u64 =
                io.DefineVariable<uint64_t>("u64", shape, start, count);
            auto var_r32 = io.DefineVariable<float>("r32", shape, start, count);
            auto var_r64 =
                io.DefineVariable<double>("r64", shape, start, count);
        }

        if (!engineName.empty())
            io.SetEngine(engineName);
        else
            io.SetEngine("HDF5");

        io.AddTransport("file");

        adios2::Engine engine = io.Open(fname, adios2::Mode::Write);

        for (size_t step = 0; step < NSteps; ++step)
        {
            // Generate test data for each process uniquely
            SmallTestData currentTestData =
                generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize);

            // Retrieve the variables that previously went out of scope
            auto var_iString = io.InquireVariable<std::string>("iString");
            auto var_i8 = io.InquireVariable<int8_t>("i8");
            auto var_i16 = io.InquireVariable<int16_t>("i16");
            auto var_i32 = io.InquireVariable<int32_t>("i32");
            auto var_i64 = io.InquireVariable<int64_t>("i64");
            auto var_u8 = io.InquireVariable<uint8_t>("u8");
            auto var_u16 = io.InquireVariable<uint16_t>("u16");
            auto var_u32 = io.InquireVariable<uint32_t>("u32");
            auto var_u64 = io.InquireVariable<uint64_t>("u64");
            auto var_r32 = io.InquireVariable<float>("r32");
            auto var_r64 = io.InquireVariable<double>("r64");

            // Make a 1D selection to describe the local dimensions of the
            // variable we write and its offsets in the global spaces
            adios2::Box<adios2::Dims> sel({mpiRank * Nx}, {Nx});

            //....EXPECT_THROW(var_iString.SetSelection(sel),
            //std::invalid_argument);
            var_i8.SetSelection(sel);
            var_i16.SetSelection(sel);
            var_i32.SetSelection(sel);
            var_i64.SetSelection(sel);
            var_u8.SetSelection(sel);
            var_u16.SetSelection(sel);
            var_u32.SetSelection(sel);
            var_u64.SetSelection(sel);
            var_r32.SetSelection(sel);
            var_r64.SetSelection(sel);

            // Write each one
            // fill in the variable with values from starting index to
            // starting index + count
            engine.BeginStep();
            engine.Put(var_iString, currentTestData.S1);
            engine.Put(var_i8, currentTestData.I8.data());
            engine.Put(var_i16, currentTestData.I16.data());
            engine.Put(var_i32, currentTestData.I32.data());
            engine.Put(var_i64, currentTestData.I64.data());
            engine.Put(var_u8, currentTestData.U8.data());
            engine.Put(var_u16, currentTestData.U16.data());
            engine.Put(var_u32, currentTestData.U32.data());
            engine.Put(var_u64, currentTestData.U64.data());
            engine.Put(var_r32, currentTestData.R32.data());
            engine.Put(var_r64, currentTestData.R64.data());
            engine.EndStep();
        }

        // Close the file
        engine.Close();
    }

    size_t ExtraSteps = 2;

    {
        // Append
        adios2::IO io = adios.DeclareIO("ioAppend");

        if (!engineName.empty())
            io.SetEngine(engineName);
        else
            io.SetEngine("HDF5");

        adios2::Engine appender = io.Open(fname, adios2::Mode::Append);

        for (size_t step = NSteps; step < NSteps + ExtraSteps; ++step)
        {
            // Generate test data for each process uniquely
            SmallTestData currentTestData =
                generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize);

            // Retrieve the variables that previously went out of scope
            auto var_iString = io.InquireVariable<std::string>("iString");
            EXPECT_TRUE(var_iString);
            auto var_i8 = io.InquireVariable<int8_t>("i8");
            EXPECT_TRUE(var_i8);
            auto var_i16 = io.InquireVariable<int16_t>("i16");
            auto var_i32 = io.InquireVariable<int32_t>("i32");
            auto var_i64 = io.InquireVariable<int64_t>("i64");
            auto var_u8 = io.InquireVariable<uint8_t>("u8");
            auto var_u16 = io.InquireVariable<uint16_t>("u16");
            auto var_u32 = io.InquireVariable<uint32_t>("u32");
            auto var_u64 = io.InquireVariable<uint64_t>("u64");
            auto var_r32 = io.InquireVariable<float>("r32");
            auto var_r64 = io.InquireVariable<double>("r64");

            // Make a 1D selection to describe the local dimensions of the
            // variable we write and its offsets in the global spaces
            adios2::Box<adios2::Dims> sel({mpiRank * Nx}, {Nx});

            EXPECT_THROW(var_iString.SetSelection(sel), std::invalid_argument);
            var_i8.SetSelection(sel);
            var_i16.SetSelection(sel);
            var_i32.SetSelection(sel);
            var_i64.SetSelection(sel);
            var_u8.SetSelection(sel);
            var_u16.SetSelection(sel);
            var_u32.SetSelection(sel);
            var_u64.SetSelection(sel);
            var_r32.SetSelection(sel);
            var_r64.SetSelection(sel);

            // Write each one
            // fill in the variable with values from starting index to
            // starting index + count
            appender.BeginStep();
            appender.Put(var_iString, currentTestData.S1);
            appender.Put(var_i8, currentTestData.I8.data());
            appender.Put(var_i16, currentTestData.I16.data());
            appender.Put(var_i32, currentTestData.I32.data());
            appender.Put(var_i64, currentTestData.I64.data());
            appender.Put(var_u8, currentTestData.U8.data());
            appender.Put(var_u16, currentTestData.U16.data());
            appender.Put(var_u32, currentTestData.U32.data());
            appender.Put(var_u64, currentTestData.U64.data());
            appender.Put(var_r32, currentTestData.R32.data());
            appender.Put(var_r64, currentTestData.R64.data());
            appender.EndStep();
        }
        appender.Close();
    }

    {
        // Read back
        adios2::IO io = adios.DeclareIO("ioRead");
        if (!engineName.empty())
            io.SetEngine(engineName);
        else
            io.SetEngine("HDF5");

        adios2::Engine reader = io.Open(fname, adios2::Mode::Read);
        EXPECT_EQ(reader.Steps(), NSteps + ExtraSteps);

        reader.Close();
    }
}

//******************************************************************************
// main
//******************************************************************************

int main(int argc, char **argv)
{
#if ADIOS2_USE_MPI
    MPI_Init(nullptr, nullptr);
#endif

    int result;
    ::testing::InitGoogleTest(&argc, argv);

    if (argc > 1)
    {
        engineName = std::string(argv[1]);
    }
    result = RUN_ALL_TESTS();

#if ADIOS2_USE_MPI
    MPI_Finalize();
#endif

    return result;
}
<commit_msg>check read results<commit_after>/*
 * Distributed under the OSI-approved Apache License, Version 2.0.  See
 * accompanying file Copyright.txt for details.
 */
#include <cstdint>
#include <cstring>

#include <iostream>
#include <stdexcept>

#include <adios2.h>

#include <gtest/gtest.h>

#include "../SmallTestData.h"

std::string engineName; // comes from command line

class AppendTimeStepTest : public ::testing::Test
{
public:
    AppendTimeStepTest() = default;

    SmallTestData m_TestData;
};

//******************************************************************************
// 1D 1x8 test data
//******************************************************************************

// ADIOS2 HDF5 write, then append, then read back
TEST_F(AppendTimeStepTest, ADIOS2HDF5WriteAppendRead)
{
    // Each process would write a 1x8 array and all processes would
    // form a mpiSize * Nx 1D array
    std::string fname = "appendTest.h5";

    int mpiRank = 0, mpiSize = 1;
    // Number of rows
    const std::size_t Nx = 8;

    // Number of steps
    const std::size_t NSteps = 3;

#if ADIOS2_USE_MPI
    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
#endif

#if ADIOS2_USE_MPI
    adios2::ADIOS adios(MPI_COMM_WORLD);
#else
    adios2::ADIOS adios;
#endif

    // Write test data using HDF5 engine
    {
        adios2::IO io = adios.DeclareIO("TestIO");

        // Declare 1D variables (NumOfProcesses * Nx)
        // The local process' part (start, count) can be defined now or later
        // before Write().
        {
            adios2::Dims shape{static_cast<unsigned int>(Nx * mpiSize)};
            adios2::Dims start{static_cast<unsigned int>(Nx * mpiRank)};
            adios2::Dims count{static_cast<unsigned int>(Nx)};

            auto var_iString = io.DefineVariable<std::string>("iString");
            auto var_i8 = io.DefineVariable<int8_t>("i8", shape, start, count);
            auto var_i16 =
                io.DefineVariable<int16_t>("i16", shape, start, count);
            auto var_i32 =
                io.DefineVariable<int32_t>("i32", shape, start, count);
            auto var_i64 =
                io.DefineVariable<int64_t>("i64", shape, start, count);
            auto var_u8 = io.DefineVariable<uint8_t>("u8", shape, start, count);
            auto var_u16 =
                io.DefineVariable<uint16_t>("u16", shape, start, count);
            auto var_u32 =
                io.DefineVariable<uint32_t>("u32", shape, start, count);
            auto var_u64 =
                io.DefineVariable<uint64_t>("u64", shape, start, count);
            auto var_r32 = io.DefineVariable<float>("r32", shape, start, count);
            auto var_r64 =
                io.DefineVariable<double>("r64", shape, start, count);
        }

        if (!engineName.empty())
            io.SetEngine(engineName);
        else
            io.SetEngine("HDF5");

        io.AddTransport("file");

        adios2::Engine engine = io.Open(fname, adios2::Mode::Write);

        for (size_t step = 0; step < NSteps; ++step)
        {
            // Generate test data for each process uniquely
            SmallTestData currentTestData =
                generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize);

            // Retrieve the variables that previously went out of scope
            auto var_iString = io.InquireVariable<std::string>("iString");
            auto var_i8 = io.InquireVariable<int8_t>("i8");
            auto var_i16 = io.InquireVariable<int16_t>("i16");
            auto var_i32 = io.InquireVariable<int32_t>("i32");
            auto var_i64 = io.InquireVariable<int64_t>("i64");
            auto var_u8 = io.InquireVariable<uint8_t>("u8");
            auto var_u16 = io.InquireVariable<uint16_t>("u16");
            auto var_u32 = io.InquireVariable<uint32_t>("u32");
            auto var_u64 = io.InquireVariable<uint64_t>("u64");
            auto var_r32 = io.InquireVariable<float>("r32");
            auto var_r64 = io.InquireVariable<double>("r64");

            // Make a 1D selection to describe the local dimensions of the
            // variable we write and its offsets in the global spaces
            adios2::Box<adios2::Dims> sel({mpiRank * Nx}, {Nx});

            var_i8.SetSelection(sel);
            var_i16.SetSelection(sel);
            var_i32.SetSelection(sel);
            var_i64.SetSelection(sel);
            var_u8.SetSelection(sel);
            var_u16.SetSelection(sel);
            var_u32.SetSelection(sel);
            var_u64.SetSelection(sel);
            var_r32.SetSelection(sel);
            var_r64.SetSelection(sel);

            // Write each one
            // fill in the variable with values from starting index to
            // starting index + count
            engine.BeginStep();
            engine.Put(var_iString, currentTestData.S1);
            engine.Put(var_i8, currentTestData.I8.data());
            engine.Put(var_i16, currentTestData.I16.data());
            engine.Put(var_i32, currentTestData.I32.data());
            engine.Put(var_i64, currentTestData.I64.data());
            engine.Put(var_u8, currentTestData.U8.data());
            engine.Put(var_u16, currentTestData.U16.data());
            engine.Put(var_u32, currentTestData.U32.data());
            engine.Put(var_u64, currentTestData.U64.data());
            engine.Put(var_r32, currentTestData.R32.data());
            engine.Put(var_r64, currentTestData.R64.data());
            engine.EndStep();
        }

        // Close the file
        engine.Close();
    }

    size_t ExtraSteps = 2;

    {
        // Append
        adios2::IO io = adios.DeclareIO("ioAppend");

        if (!engineName.empty())
            io.SetEngine(engineName);
        else
            io.SetEngine("HDF5");

        adios2::Engine appender = io.Open(fname, adios2::Mode::Append);

        for (size_t step = NSteps; step < NSteps + ExtraSteps; ++step)
        {
            // Generate test data for each process uniquely
            SmallTestData currentTestData =
                generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize);

            // Retrieve the variables that previously went out of scope
            auto var_iString = io.InquireVariable<std::string>("iString");
            EXPECT_TRUE(var_iString);
            auto var_i8 = io.InquireVariable<int8_t>("i8");
            EXPECT_TRUE(var_i8);
            auto var_i16 = io.InquireVariable<int16_t>("i16");
            auto var_i32 = io.InquireVariable<int32_t>("i32");
            auto var_i64 = io.InquireVariable<int64_t>("i64");
            auto var_u8 = io.InquireVariable<uint8_t>("u8");
            auto var_u16 = io.InquireVariable<uint16_t>("u16");
            auto var_u32 = io.InquireVariable<uint32_t>("u32");
            auto var_u64 = io.InquireVariable<uint64_t>("u64");
            auto var_r32 = io.InquireVariable<float>("r32");
            auto var_r64 = io.InquireVariable<double>("r64");

            // Make a 1D selection to describe the local dimensions of the
            // variable we write and its offsets in the global spaces
            adios2::Box<adios2::Dims> sel({mpiRank * Nx}, {Nx});

            EXPECT_THROW(var_iString.SetSelection(sel), std::invalid_argument);
            var_i8.SetSelection(sel);
            var_i16.SetSelection(sel);
            var_i32.SetSelection(sel);
            var_i64.SetSelection(sel);
            var_u8.SetSelection(sel);
            var_u16.SetSelection(sel);
            var_u32.SetSelection(sel);
            var_u64.SetSelection(sel);
            var_r32.SetSelection(sel);
            var_r64.SetSelection(sel);

            // Write each one
            // fill in the variable with values from starting index to
            // starting index + count
            appender.BeginStep();
            appender.Put(var_iString, currentTestData.S1);
            appender.Put(var_i8, currentTestData.I8.data());
            appender.Put(var_i16, currentTestData.I16.data());
            appender.Put(var_i32, currentTestData.I32.data());
            appender.Put(var_i64, currentTestData.I64.data());
            appender.Put(var_u8, currentTestData.U8.data());
            appender.Put(var_u16, currentTestData.U16.data());
            appender.Put(var_u32, currentTestData.U32.data());
            appender.Put(var_u64, currentTestData.U64.data());
            appender.Put(var_r32, currentTestData.R32.data());
            appender.Put(var_r64, currentTestData.R64.data());
            appender.EndStep();
        }
        appender.Close();
    }

    {
        // Read back
        adios2::IO io = adios.DeclareIO("ioRead");
        if (!engineName.empty())
            io.SetEngine(engineName);
        else
            io.SetEngine("HDF5");

        adios2::Engine reader = io.Open(fname, adios2::Mode::Read);
        EXPECT_EQ(reader.Steps(), NSteps + ExtraSteps);

        std::string IString;
        std::array<int8_t, Nx> I8;
        std::array<int16_t, Nx> I16;
        std::array<int32_t, Nx> I32;
        std::array<int64_t, Nx> I64;
        std::array<uint8_t, Nx> U8;
        std::array<uint16_t, Nx> U16;
        std::array<uint32_t, Nx> U32;
        std::array<uint64_t, Nx> U64;
        std::array<float, Nx> R32;
        std::array<double, Nx> R64;

        auto var_iString = io.InquireVariable<std::string>("iString");
        EXPECT_TRUE(var_iString);
        EXPECT_EQ(var_iString.Steps(), NSteps + ExtraSteps);

        auto var_i8 = io.InquireVariable<int8_t>("i8");
        EXPECT_TRUE(var_i8);
        EXPECT_EQ(var_i8.Steps(), NSteps + ExtraSteps);

        auto var_i16 = io.InquireVariable<int16_t>("i16");
        EXPECT_TRUE(var_i16);
        EXPECT_EQ(var_i16.Steps(), NSteps + ExtraSteps);

        auto var_i32 = io.InquireVariable<int32_t>("i32");
        EXPECT_TRUE(var_i32);
        EXPECT_EQ(var_i32.Steps(), NSteps + ExtraSteps);

        auto var_i64 = io.InquireVariable<int64_t>("i64");
        EXPECT_TRUE(var_i64);
        EXPECT_EQ(var_i64.Steps(), NSteps + ExtraSteps);

        auto var_u8 = io.InquireVariable<uint8_t>("u8");
        EXPECT_TRUE(var_u8);
        EXPECT_EQ(var_u8.Steps(), NSteps + ExtraSteps);

        auto var_u16 = io.InquireVariable<uint16_t>("u16");
        EXPECT_TRUE(var_u16);
        EXPECT_EQ(var_u16.Steps(), NSteps + ExtraSteps);

        auto var_u32 = io.InquireVariable<uint32_t>("u32");
        EXPECT_TRUE(var_u32);
        EXPECT_EQ(var_u32.Steps(), NSteps + ExtraSteps);

        auto var_u64 = io.InquireVariable<uint64_t>("u64");
        EXPECT_TRUE(var_u64);
        EXPECT_EQ(var_u64.Steps(), NSteps + ExtraSteps);

        auto var_r32 = io.InquireVariable<float>("r32");
        EXPECT_TRUE(var_r32);
        EXPECT_EQ(var_r32.Steps(), NSteps + ExtraSteps);

        auto var_r64 = io.InquireVariable<double>("r64");
        EXPECT_TRUE(var_r64);
        EXPECT_EQ(var_r64.Steps(), NSteps + ExtraSteps);

        adios2::Box<adios2::Dims> sel({mpiRank * Nx}, {Nx});

        for (size_t step = 0; step < NSteps + ExtraSteps; ++step)
        {
            SmallTestData currentTestData =
                generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize);

            var_i8.SetStepSelection({step, 1});
            var_i8.SetSelection(sel);
            reader.Get(var_i8, I8.data());

            var_i16.SetStepSelection({step, 1});
            var_i16.SetSelection(sel);
            reader.Get(var_i16, I16.data());

            var_i32.SetStepSelection({step, 1});
            var_i32.SetSelection(sel);
            reader.Get(var_i32, I32.data());

            var_i64.SetStepSelection({step, 1});
            var_i64.SetSelection(sel);
            reader.Get(var_i64, I64.data());

            var_u8.SetStepSelection({step, 1});
            var_u8.SetSelection(sel);
            reader.Get(var_u8, U8.data());

            var_u16.SetStepSelection({step, 1});
            var_u16.SetSelection(sel);
            reader.Get(var_u16, U16.data());

            var_u32.SetStepSelection({step, 1});
            var_u32.SetSelection(sel);
            reader.Get(var_u32, U32.data());

            var_u64.SetStepSelection({step, 1});
            var_u64.SetSelection(sel);
            reader.Get(var_u64, U64.data());

            var_r32.SetStepSelection({step, 1});
            var_r32.SetSelection(sel);
            reader.Get(var_r32, R32.data());

            var_r64.SetStepSelection({step, 1});
            var_r64.SetSelection(sel);
            reader.Get(var_r64, R64.data());

            reader.Get(var_iString, IString);
            reader.PerformGets();

            EXPECT_EQ(IString, currentTestData.S1);

            for (size_t i = 0; i < Nx; ++i)
            {
                std::stringstream ss;
                ss << "step=" << step << " i=" << i << " rank=" << mpiRank;
                std::string msg = ss.str();

                EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg;
                EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg;
                EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg;
                EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg;

                EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg;
                EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg;
                EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg;
                EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg;

                EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg;
                EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg;
            }
        }

        reader.Close();
    }
}

//******************************************************************************
// main
//******************************************************************************

int main(int argc, char **argv)
{
#if ADIOS2_USE_MPI
    MPI_Init(nullptr, nullptr);
#endif

    int result;
    ::testing::InitGoogleTest(&argc, argv);

    if (argc > 1)
    {
        engineName = std::string(argv[1]);
    }
    result = RUN_ALL_TESTS();

#if ADIOS2_USE_MPI
    MPI_Finalize();
#endif

    return result;
}
<|endoftext|>


<commit_before>#include <utAlgorithm/3DPointReconstruction.h>
#include <utMath/Geometry/PointProjection.h>
#include <utMath/Stochastic/identity_iterator.h>

#include <utMath/Random/Scalar.h>
#include <utMath/Random/Vector.h>
#include <utMath/Random/Rotation.h>
#include "../tools.h"

#include <vector>
#include <iostream>

#include <boost/test/unit_test.hpp>
#include <boost/test/floating_point_comparison.hpp>

using namespace Ubitrack;

template< typename T >
void Test2Cameras( const std::size_t n_runs, const T epsilon )
{
	typename Math::Random::Quaternion< T >::Uniform randQuat;
	typename Math::Random::Vector< T, 3 >::Uniform randTranslation( -10., 10. ); //translation

	typename Math::Random::Vector< T, 3 >::Uniform randVector( -1., 1. ); // 3d Points

	for( std::size_t j=0; j<n_runs; ++j )
	{

		Math::Pose CamPose1( randQuat() , randTranslation() );
		Math::Pose CamPose2( randQuat() , randTranslation() );
		Math::Matrix< T, 3, 4 > proj1( CamPose1 );
		Math::Matrix< T, 3, 4 > proj2( CamPose2 );

		Math::Matrix< T, 3, 3 > K = Math::Matrix< T, 3, 3 >::identity();
		K( 0, 0 ) = K( 1, 1 ) = 500;
		K( 0, 2 ) = 320;
		K( 1, 2 ) = 240;

		// generate the projections
		proj1 = boost::numeric::ublas::prod( K, proj1 );
		proj2 = boost::numeric::ublas::prod( K, proj2 );

		//compute random points
		const std::size_t n( Math::Random::distribute_uniform< std::size_t >( 10, 30 ) );
		std::vector< Math::Vector< T, 3 > > objPoints;
		objPoints.reserve( n );
		std::generate_n ( std::back_inserter( objPoints ), n,  randVector );

		//project the points onto the first image screen
		std::vector< Math::Vector< T, 2 > > points1;
		points1.reserve( n );
		Math::Geometry::project_points( proj1, objPoints.begin(), objPoints.end(), std::back_inserter( points1 ) );

		//project the points onto the second image screen
		std::vector< Math::Vector< T, 2 > > points2;
		points2.reserve( n );
		Math::Geometry::project_points( proj2, objPoints.begin(), objPoints.end(), std::back_inserter( points2 ) );

		for( std::size_t i=0; i<n; ++i )
		{
			//check now the reconstructed points
			Math::Vector< T, 3 > p3D = Algorithm::get3DPosition( proj1, proj2, points1[ i ], points2[ i ] );
			const T diffError = vectorDiff( p3D, objPoints[ i ] );
			BOOST_CHECK( diffError < epsilon );
		}
	}
}

template< typename T >
void TestMulitpleCameras( const std::size_t n_runs , const T epsilon )
{
	typename Math::Random::Quaternion< T >::Uniform randQuat;
	typename Math::Random::Vector< T, 3 >::Uniform randTranslation( -10., 10. ); //translation

	typename Math::Random::Vector< T, 3 >::Uniform randVector( -1., 1. ); // 3d Points
	// typename Random::Vector< T, 3 >::Normal randPositionNoise( 0, 0.2 ); // translation gaussian noise

	// check for multi-camera setup
	for( std::size_t j=0; j<n_runs; ++j )
	{
		// determine the amount of cameras to be used
		const std::size_t n_cams( Math::Random::distribute_uniform< std::size_t >( 2, 10 ) );

		// intialize some cameras:
		std::vector< Math::Matrix< T, 3, 4 > > matrices; // stores projection matrices
		matrices.reserve( n_cams );

		for( std::size_t i( 0 ); i < n_cams; ++i )
		{
			Math::Pose CamPose( randQuat() , randTranslation() );
			Math::Matrix< T, 3, 4 > p( CamPose );
			matrices.push_back( p );
		}

		// generate a random vector
		const Math::Vector< T, 3 >  randVec = randVector();

		// generate some 2D observation of 3D random vector
		std::vector< Math::Vector< T, 2 > > pts; //stores image points
		pts.reserve( n_cams );
		std::transform( matrices.begin(), matrices.end(), Util::identity< Math::Vector< T, 3 > >( randVec ).begin()
			, std::back_inserter( pts ), Math::Geometry::ProjectPoint() );

		//estimate final result
		Math::Vector< T, 3 > p3D = Algorithm::get3DPosition( matrices, pts, 0 );
		const T diffError = vectorDiff( p3D, randVec );
		BOOST_CHECK_SMALL( diffError, epsilon );
	}
}

void Test3DPointReconstruction()
{
	Test2Cameras< float >( 1000, 1e-2f );
	Test2Cameras< double >( 1000, 1e-3 );
	TestMulitpleCameras< float >( 1000, 1e-2f );
	TestMulitpleCameras< double >( 1000, 1e-3 );
}<commit_msg>added constness to also remove compile problem in debug mode here<commit_after>#include <utAlgorithm/3DPointReconstruction.h>
#include <utMath/Geometry/PointProjection.h>
#include <utMath/Stochastic/identity_iterator.h>

#include <utMath/Random/Scalar.h>
#include <utMath/Random/Vector.h>
#include <utMath/Random/Rotation.h>
#include "../tools.h"

#include <vector>
#include <iostream>

#include <boost/test/unit_test.hpp>
#include <boost/test/floating_point_comparison.hpp>

using namespace Ubitrack;

template< typename T >
void Test2Cameras( const std::size_t n_runs, const T epsilon )
{
	typename Math::Random::Quaternion< T >::Uniform randQuat;
	typename Math::Random::Vector< T, 3 >::Uniform randTranslation( -10., 10. ); //translation

	typename Math::Random::Vector< T, 3 >::Uniform randVector( -1., 1. ); // 3d Points

	for( std::size_t j=0; j<n_runs; ++j )
	{

		Math::Pose CamPose1( randQuat() , randTranslation() );
		Math::Pose CamPose2( randQuat() , randTranslation() );
		Math::Matrix< T, 3, 4 > proj1( CamPose1 );
		Math::Matrix< T, 3, 4 > proj2( CamPose2 );

		Math::Matrix< T, 3, 3 > K = Math::Matrix< T, 3, 3 >::identity();
		K( 0, 0 ) = K( 1, 1 ) = 500;
		K( 0, 2 ) = 320;
		K( 1, 2 ) = 240;

		// generate the projections
		proj1 = boost::numeric::ublas::prod( K, proj1 );
		proj2 = boost::numeric::ublas::prod( K, proj2 );

		//compute random points
		const std::size_t n( Math::Random::distribute_uniform< std::size_t >( 10, 30 ) );
		std::vector< Math::Vector< T, 3 > > objPoints;
		objPoints.reserve( n );
		std::generate_n ( std::back_inserter( objPoints ), n,  randVector );

		//project the points onto the first image screen
		std::vector< Math::Vector< T, 2 > > points1;
		points1.reserve( n );
		Math::Geometry::project_points( proj1, objPoints.begin(), objPoints.end(), std::back_inserter( points1 ) );

		//project the points onto the second image screen
		std::vector< Math::Vector< T, 2 > > points2;
		points2.reserve( n );
		Math::Geometry::project_points( proj2, objPoints.begin(), objPoints.end(), std::back_inserter( points2 ) );

		for( std::size_t i=0; i<n; ++i )
		{
			//check now the reconstructed points
			Math::Vector< T, 3 > p3D = Algorithm::get3DPosition( proj1, proj2, points1[ i ], points2[ i ] );
			const T diffError = vectorDiff( p3D, objPoints[ i ] );
			BOOST_CHECK( diffError < epsilon );
		}
	}
}

template< typename T >
void TestMulitpleCameras( const std::size_t n_runs , const T epsilon )
{
	typename Math::Random::Quaternion< T >::Uniform randQuat;
	typename Math::Random::Vector< T, 3 >::Uniform randTranslation( -10., 10. ); //translation

	typename Math::Random::Vector< T, 3 >::Uniform randVector( -1., 1. ); // 3d Points
	// typename Random::Vector< T, 3 >::Normal randPositionNoise( 0, 0.2 ); // translation gaussian noise

	// check for multi-camera setup
	for( std::size_t j=0; j<n_runs; ++j )
	{
		// determine the amount of cameras to be used
		const std::size_t n_cams( Math::Random::distribute_uniform< std::size_t >( 2, 10 ) );

		// initialize some cameras:
		std::vector< Math::Matrix< T, 3, 4 > > matrices; // stores projection matrices
		matrices.reserve( n_cams );

		for( std::size_t i( 0 ); i < n_cams; ++i )
		{
			Math::Pose CamPose( randQuat() , randTranslation() );
			Math::Matrix< T, 3, 4 > p( CamPose );
			matrices.push_back( p );
		}

		// generate a random vector
		const Math::Vector< T, 3 >  randVec = randVector();

		// generate some 2D observation of 3D random vector
		std::vector< Math::Vector< T, 2 > > pts; //stores image points
		pts.reserve( n_cams );
		std::transform( matrices.begin(), matrices.end(), Util::identity< const Math::Vector< T, 3 > >( randVec ).begin()
			, std::back_inserter( pts ), Math::Geometry::ProjectPoint() );

		//estimate final result
		Math::Vector< T, 3 > p3D = Algorithm::get3DPosition( matrices, pts, 0 );
		const T diffError = vectorDiff( p3D, randVec );
		BOOST_CHECK_SMALL( diffError, epsilon );
	}
}

void Test3DPointReconstruction()
{
	Test2Cameras< float >( 1000, 1e-2f );
	Test2Cameras< double >( 1000, 1e-3 );
	TestMulitpleCameras< float >( 1000, 1e-2f );
	TestMulitpleCameras< double >( 1000, 1e-3 );
}<|endoftext|>


<commit_before>#include "Players/Game_Tree_Node_Result.h"

#include <cmath>
#include <utility>
#include <limits>

#include "Game/Color.h"
#include "Utility/Math.h"

const double Game_Tree_Node_Result::win_score = std::numeric_limits<double>::infinity();
const double Game_Tree_Node_Result::lose_score = -win_score;

double Game_Tree_Node_Result::corrected_score(Color query) const
{
    return query == perspective ? score : -score;
}

std::pair<double, int> Game_Tree_Node_Result::value(Color query) const
{
    auto standardized_score = corrected_score(query);
    if(std::isinf(standardized_score))
    {
        // standardized_score == +infinity means a shallower depth
        // is better, and vice versa for -infinity,
        // so make the depth the opposite sign of the score
        return {standardized_score,
                -Math::sign(standardized_score)*int(depth())};
    }
    else
    {
        // For non-winning results, the depth doesn't matter
        return {standardized_score, 0};
    }
}

size_t Game_Tree_Node_Result::depth() const
{
    return variation.size();
}

bool Game_Tree_Node_Result::is_winning_for(Color query) const
{
    return std::isinf(score) && ((score > 0) == (query == perspective));
}

bool Game_Tree_Node_Result::is_losing_for(Color query) const
{
    return std::isinf(score) && ((score < 0) == (query == perspective));
}
<commit_msg>Simplify Game_Tree_Node_Result::is_losing_for()<commit_after>#include "Players/Game_Tree_Node_Result.h"

#include <cmath>
#include <utility>
#include <limits>

#include "Game/Color.h"
#include "Utility/Math.h"

const double Game_Tree_Node_Result::win_score = std::numeric_limits<double>::infinity();
const double Game_Tree_Node_Result::lose_score = -win_score;

double Game_Tree_Node_Result::corrected_score(Color query) const
{
    return query == perspective ? score : -score;
}

std::pair<double, int> Game_Tree_Node_Result::value(Color query) const
{
    auto standardized_score = corrected_score(query);
    if(std::isinf(standardized_score))
    {
        // standardized_score == +infinity means a shallower depth
        // is better, and vice versa for -infinity,
        // so make the depth the opposite sign of the score
        return {standardized_score,
                -Math::sign(standardized_score)*int(depth())};
    }
    else
    {
        // For non-winning results, the depth doesn't matter
        return {standardized_score, 0};
    }
}

size_t Game_Tree_Node_Result::depth() const
{
    return variation.size();
}

bool Game_Tree_Node_Result::is_winning_for(Color query) const
{
    return std::isinf(score) && ((score > 0) == (query == perspective));
}

bool Game_Tree_Node_Result::is_losing_for(Color query) const
{
    return is_winning_for(opposite(query));
}
<|endoftext|>