<commit_before>#include "xchainer/context.h"

#include <cstdlib>
#include <future>

#include <gtest/gtest.h>
#include <nonstd/optional.hpp>

#include "xchainer/backend.h"
#include "xchainer/device.h"
#include "xchainer/native/native_backend.h"
#include "xchainer/native/native_device.h"
#include "xchainer/testing/threading.h"

namespace xchainer {
namespace {

TEST(ContextTest, Ctor) {
    Context();  // no throw
}

TEST(ContextTest, GetBackend) {
    Context ctx;
    Backend& backend = ctx.GetBackend("native");
    EXPECT_EQ(&backend, &ctx.GetBackend("native"));
}

TEST(ContextTest, NativeBackend) {
    Context ctx;
    native::NativeBackend& backend = ctx.GetNativeBackend();
    EXPECT_EQ(&ctx.GetBackend("native"), &backend);
}

TEST(ContextTest, GetBackendThreadSafe) {
    static constexpr int kRepeat = 100;
    static constexpr size_t kThreadCount = 1024;

    xchainer::testing::CheckThreadSafety(
            kRepeat,
            kThreadCount,
            [](size_t /*repeat*/) { return std::make_unique<Context>(); },
            [](size_t /*thread_index*/, const std::unique_ptr<Context>& ctx) {
                Backend& backend = ctx->GetBackend("native");
                return &backend;
            },
            [this](const std::vector<Backend*>& results) {
                for (Backend* backend : results) {
                    ASSERT_EQ("native", backend->GetName());
                    ASSERT_EQ(backend, results.front());
                }
            });
}

TEST(ContextTest, BackendNotFound) {
    Context ctx;
    EXPECT_THROW(ctx.GetBackend("something_that_does_not_exist"), BackendError);
}

TEST(ContextTest, GetDevice) {
    Context ctx;
    Device& device = ctx.GetDevice({"native", 0});
    EXPECT_EQ(&device, &ctx.GetDevice({"native:0"}));
}

TEST(ContextTest, GetDeviceThreadSafe) {
    static constexpr int kRepeat = 100;
    static constexpr int kDeviceCount = 4;
    static constexpr size_t kThreadCountPerDevice = 32;
    static constexpr size_t kThreadCount = kDeviceCount * kThreadCountPerDevice;

    xchainer::testing::CheckThreadSafety(
            kRepeat,
            kThreadCount,
            [](size_t /*repeat*/) { return std::make_unique<Context>(); },
            [](size_t thread_index, const std::unique_ptr<Context>& ctx) {
                int device_index = thread_index / kThreadCountPerDevice;
                Device& device = ctx->GetDevice({"native", device_index});
                return &device;
            },
            [this](const std::vector<Device*>& results) {
                // Check device pointers are identical within each set of threads corresponding to one device
                for (int device_index = 0; device_index < kDeviceCount; ++device_index) {
                    auto it_first = std::next(results.begin(), device_index * kThreadCountPerDevice);
                    auto it_last = std::next(results.begin(), (device_index + 1) * kThreadCountPerDevice);
                    Device* ref_device = *it_first;

                    // Check the device index
                    ASSERT_EQ(device_index, ref_device->index());

                    for (auto it = it_first; it != it_last; ++it) {
                        ASSERT_EQ(ref_device, *it);
                    }
                }
            });
}

TEST(ContextTest, DefaultContext) {
    SetGlobalDefaultContext(nullptr);
    SetDefaultContext(nullptr);
    ASSERT_THROW(GetDefaultContext(), XchainerError);

    Context ctx;
    SetGlobalDefaultContext(nullptr);
    SetDefaultContext(&ctx);
    ASSERT_EQ(&ctx, &GetDefaultContext());

    Context global_ctx;
    SetGlobalDefaultContext(&global_ctx);
    SetDefaultContext(nullptr);
    ASSERT_EQ(&global_ctx, &GetDefaultContext());

    SetGlobalDefaultContext(&global_ctx);
    SetDefaultContext(&ctx);
    ASSERT_EQ(&ctx, &GetDefaultContext());
}

TEST(ContextTest, GlobalDefaultContext) {
    SetGlobalDefaultContext(nullptr);
    ASSERT_THROW(GetGlobalDefaultContext(), XchainerError);

    Context ctx;
    SetGlobalDefaultContext(&ctx);
    ASSERT_EQ(&ctx, &GetGlobalDefaultContext());
}

TEST(ContextTest, ThreadLocal) {
    Context ctx;
    SetDefaultContext(&ctx);

    Context ctx2;
    auto future = std::async(std::launch::async, [&ctx2] {
        SetDefaultContext(&ctx2);
        return &GetDefaultContext();
    });
    ASSERT_NE(&GetDefaultContext(), future.get());
}

TEST(ContextTest, ContextScopeCtor) {
    SetGlobalDefaultContext(nullptr);
    SetDefaultContext(nullptr);
    Context ctx1;
    {
        // ContextScope should work even if default context is not set
        ContextScope scope(ctx1);
        EXPECT_EQ(&ctx1, &GetDefaultContext());
    }
    ASSERT_THROW(GetDefaultContext(), XchainerError);
    SetDefaultContext(&ctx1);
    {
        Context ctx2;
        ContextScope scope(ctx2);
        EXPECT_EQ(&ctx2, &GetDefaultContext());
    }
    ASSERT_EQ(&ctx1, &GetDefaultContext());
    {
        ContextScope scope;
        EXPECT_EQ(&ctx1, &GetDefaultContext());
        Context ctx2;
        SetDefaultContext(&ctx2);
    }
    ASSERT_EQ(&ctx1, &GetDefaultContext());
    Context ctx2;
    {
        ContextScope scope(ctx2);
        scope.Exit();
        EXPECT_EQ(&ctx1, &GetDefaultContext());
        SetDefaultContext(&ctx2);
        // not recovered here because the scope has already existed
    }
    ASSERT_EQ(&ctx2, &GetDefaultContext());
}

TEST(ContextTest, ContextScopeResetDevice) {
    SetGlobalDefaultContext(nullptr);
    SetDefaultContext(nullptr);
    Context ctx1;
    Context ctx2;
    {
        ContextScope ctx_scope1{ctx1};
        Device& device1 = ctx1.GetDevice({"native", 0});
        DeviceScope dev_scope1{device1};

        {
            ContextScope ctx_scope2{ctx2};
            ASSERT_NE(&device1, &GetDefaultDevice());
            Device& device2 = ctx2.GetDevice({"native", 0});
            SetDefaultDevice(&device2);
        }

        EXPECT_EQ(&device1, &GetDefaultDevice());
    }
}

TEST(ContextTest, UserDefinedBackend) {
    ::setenv("XCHAINER_PATH", XCHAINER_TEST_DIR "/context_testdata", 1);
    Context ctx;
    Backend& backend0 = ctx.GetBackend("backend0");
    EXPECT_EQ("backend0", backend0.GetName());
    Backend& backend0_2 = ctx.GetBackend("backend0");
    EXPECT_EQ(&backend0, &backend0_2);
    Backend& backend1 = ctx.GetBackend("backend1");
    EXPECT_EQ("backend1", backend1.GetName());

    Device& device0 = ctx.GetDevice(std::string("backend0:0"));
    EXPECT_EQ(&backend0, &device0.backend());
}

TEST(ContextTest, GetBackendOnDefaultContext) {
    // xchainer::GetBackend
    Context ctx;
    SetDefaultContext(&ctx);
    Backend& backend = GetBackend("native");
    EXPECT_EQ(&ctx, &backend.context());
    EXPECT_EQ("native", backend.GetName());
}

TEST(ContextTest, GetNativeBackendOnDefaultContext) {
    // xchainer::GetNativeBackend
    Context ctx;
    SetDefaultContext(&ctx);
    native::NativeBackend& backend = GetNativeBackend();
    EXPECT_EQ(&ctx.GetNativeBackend(), &backend);
}

TEST(ContextTest, GetDeviceOnDefaultContext) {
    // xchainer::GetDevice
    Context ctx;
    SetDefaultContext(&ctx);
    Device& device = GetDevice({"native:0"});
    EXPECT_EQ(&ctx, &device.backend().context());
    EXPECT_EQ("native:0", device.name());
}

}  // namespace
}  // namespace xchainer
<commit_msg>int -> size_t<commit_after>#include "xchainer/context.h"

#include <cstdlib>
#include <future>

#include <gtest/gtest.h>
#include <nonstd/optional.hpp>

#include "xchainer/backend.h"
#include "xchainer/device.h"
#include "xchainer/native/native_backend.h"
#include "xchainer/native/native_device.h"
#include "xchainer/testing/threading.h"

namespace xchainer {
namespace {

TEST(ContextTest, Ctor) {
    Context();  // no throw
}

TEST(ContextTest, GetBackend) {
    Context ctx;
    Backend& backend = ctx.GetBackend("native");
    EXPECT_EQ(&backend, &ctx.GetBackend("native"));
}

TEST(ContextTest, NativeBackend) {
    Context ctx;
    native::NativeBackend& backend = ctx.GetNativeBackend();
    EXPECT_EQ(&ctx.GetBackend("native"), &backend);
}

TEST(ContextTest, GetBackendThreadSafe) {
    static constexpr size_t kRepeat = 100;
    static constexpr size_t kThreadCount = 1024;

    xchainer::testing::CheckThreadSafety(
            kRepeat,
            kThreadCount,
            [](size_t /*repeat*/) { return std::make_unique<Context>(); },
            [](size_t /*thread_index*/, const std::unique_ptr<Context>& ctx) {
                Backend& backend = ctx->GetBackend("native");
                return &backend;
            },
            [this](const std::vector<Backend*>& results) {
                for (Backend* backend : results) {
                    ASSERT_EQ("native", backend->GetName());
                    ASSERT_EQ(backend, results.front());
                }
            });
}

TEST(ContextTest, BackendNotFound) {
    Context ctx;
    EXPECT_THROW(ctx.GetBackend("something_that_does_not_exist"), BackendError);
}

TEST(ContextTest, GetDevice) {
    Context ctx;
    Device& device = ctx.GetDevice({"native", 0});
    EXPECT_EQ(&device, &ctx.GetDevice({"native:0"}));
}

TEST(ContextTest, GetDeviceThreadSafe) {
    static constexpr size_t kRepeat = 100;
    static constexpr int kDeviceCount = 4;
    static constexpr size_t kThreadCountPerDevice = 32;
    static constexpr size_t kThreadCount = kDeviceCount * kThreadCountPerDevice;

    xchainer::testing::CheckThreadSafety(
            kRepeat,
            kThreadCount,
            [](size_t /*repeat*/) { return std::make_unique<Context>(); },
            [](size_t thread_index, const std::unique_ptr<Context>& ctx) {
                int device_index = thread_index / kThreadCountPerDevice;
                Device& device = ctx->GetDevice({"native", device_index});
                return &device;
            },
            [this](const std::vector<Device*>& results) {
                // Check device pointers are identical within each set of threads corresponding to one device
                for (int device_index = 0; device_index < kDeviceCount; ++device_index) {
                    auto it_first = std::next(results.begin(), device_index * kThreadCountPerDevice);
                    auto it_last = std::next(results.begin(), (device_index + 1) * kThreadCountPerDevice);
                    Device* ref_device = *it_first;

                    // Check the device index
                    ASSERT_EQ(device_index, ref_device->index());

                    for (auto it = it_first; it != it_last; ++it) {
                        ASSERT_EQ(ref_device, *it);
                    }
                }
            });
}

TEST(ContextTest, DefaultContext) {
    SetGlobalDefaultContext(nullptr);
    SetDefaultContext(nullptr);
    ASSERT_THROW(GetDefaultContext(), XchainerError);

    Context ctx;
    SetGlobalDefaultContext(nullptr);
    SetDefaultContext(&ctx);
    ASSERT_EQ(&ctx, &GetDefaultContext());

    Context global_ctx;
    SetGlobalDefaultContext(&global_ctx);
    SetDefaultContext(nullptr);
    ASSERT_EQ(&global_ctx, &GetDefaultContext());

    SetGlobalDefaultContext(&global_ctx);
    SetDefaultContext(&ctx);
    ASSERT_EQ(&ctx, &GetDefaultContext());
}

TEST(ContextTest, GlobalDefaultContext) {
    SetGlobalDefaultContext(nullptr);
    ASSERT_THROW(GetGlobalDefaultContext(), XchainerError);

    Context ctx;
    SetGlobalDefaultContext(&ctx);
    ASSERT_EQ(&ctx, &GetGlobalDefaultContext());
}

TEST(ContextTest, ThreadLocal) {
    Context ctx;
    SetDefaultContext(&ctx);

    Context ctx2;
    auto future = std::async(std::launch::async, [&ctx2] {
        SetDefaultContext(&ctx2);
        return &GetDefaultContext();
    });
    ASSERT_NE(&GetDefaultContext(), future.get());
}

TEST(ContextTest, ContextScopeCtor) {
    SetGlobalDefaultContext(nullptr);
    SetDefaultContext(nullptr);
    Context ctx1;
    {
        // ContextScope should work even if default context is not set
        ContextScope scope(ctx1);
        EXPECT_EQ(&ctx1, &GetDefaultContext());
    }
    ASSERT_THROW(GetDefaultContext(), XchainerError);
    SetDefaultContext(&ctx1);
    {
        Context ctx2;
        ContextScope scope(ctx2);
        EXPECT_EQ(&ctx2, &GetDefaultContext());
    }
    ASSERT_EQ(&ctx1, &GetDefaultContext());
    {
        ContextScope scope;
        EXPECT_EQ(&ctx1, &GetDefaultContext());
        Context ctx2;
        SetDefaultContext(&ctx2);
    }
    ASSERT_EQ(&ctx1, &GetDefaultContext());
    Context ctx2;
    {
        ContextScope scope(ctx2);
        scope.Exit();
        EXPECT_EQ(&ctx1, &GetDefaultContext());
        SetDefaultContext(&ctx2);
        // not recovered here because the scope has already existed
    }
    ASSERT_EQ(&ctx2, &GetDefaultContext());
}

TEST(ContextTest, ContextScopeResetDevice) {
    SetGlobalDefaultContext(nullptr);
    SetDefaultContext(nullptr);
    Context ctx1;
    Context ctx2;
    {
        ContextScope ctx_scope1{ctx1};
        Device& device1 = ctx1.GetDevice({"native", 0});
        DeviceScope dev_scope1{device1};

        {
            ContextScope ctx_scope2{ctx2};
            ASSERT_NE(&device1, &GetDefaultDevice());
            Device& device2 = ctx2.GetDevice({"native", 0});
            SetDefaultDevice(&device2);
        }

        EXPECT_EQ(&device1, &GetDefaultDevice());
    }
}

TEST(ContextTest, UserDefinedBackend) {
    ::setenv("XCHAINER_PATH", XCHAINER_TEST_DIR "/context_testdata", 1);
    Context ctx;
    Backend& backend0 = ctx.GetBackend("backend0");
    EXPECT_EQ("backend0", backend0.GetName());
    Backend& backend0_2 = ctx.GetBackend("backend0");
    EXPECT_EQ(&backend0, &backend0_2);
    Backend& backend1 = ctx.GetBackend("backend1");
    EXPECT_EQ("backend1", backend1.GetName());

    Device& device0 = ctx.GetDevice(std::string("backend0:0"));
    EXPECT_EQ(&backend0, &device0.backend());
}

TEST(ContextTest, GetBackendOnDefaultContext) {
    // xchainer::GetBackend
    Context ctx;
    SetDefaultContext(&ctx);
    Backend& backend = GetBackend("native");
    EXPECT_EQ(&ctx, &backend.context());
    EXPECT_EQ("native", backend.GetName());
}

TEST(ContextTest, GetNativeBackendOnDefaultContext) {
    // xchainer::GetNativeBackend
    Context ctx;
    SetDefaultContext(&ctx);
    native::NativeBackend& backend = GetNativeBackend();
    EXPECT_EQ(&ctx.GetNativeBackend(), &backend);
}

TEST(ContextTest, GetDeviceOnDefaultContext) {
    // xchainer::GetDevice
    Context ctx;
    SetDefaultContext(&ctx);
    Device& device = GetDevice({"native:0"});
    EXPECT_EQ(&ctx, &device.backend().context());
    EXPECT_EQ("native:0", device.name());
}

}  // namespace
}  // namespace xchainer
<|endoftext|>


<commit_before>// __BEGIN_LICENSE__
//  Copyright (c) 2009-2012, United States Government as represented by the
//  Administrator of the National Aeronautics and Space Administration. All
//  rights reserved.
//
//  The NGT platform is licensed under the Apache License, Version 2.0 (the
//  "License"); you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//  http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
// __END_LICENSE__

/// \file dem_adjust.cc
///

#include <vw/FileIO.h>
#include <vw/Image.h>
#include <vw/Cartography.h>
#include <vw/Math.h>
#include <asp/Core/Macros.h>
#include <asp/Core/Common.h>

namespace po = boost::program_options;
namespace fs = boost::filesystem;

using std::endl;
using std::string;
using namespace vw;
using namespace vw::cartography;

template <class ImageT>
class AdjustDemView : public ImageViewBase<AdjustDemView<ImageT> >
{
  ImageT m_img;
  GeoReference const& m_georef;
  ImageViewRef<PixelMask<double> > const& m_delta;
  GeoReference const& m_delta_georef;
  double m_nodata_val;

public:

  typedef double pixel_type;
  typedef double result_type;
  typedef ProceduralPixelAccessor<AdjustDemView> pixel_accessor;

  AdjustDemView(ImageT const& img, GeoReference const& georef,
                ImageViewRef<PixelMask<double> > const& delta,
                GeoReference const& delta_georef, double nodata_val):
    m_img(img), m_georef(georef),
    m_delta(delta), m_delta_georef(delta_georef),
    m_nodata_val(nodata_val){}

  inline int32 cols() const { return m_img.cols(); }
  inline int32 rows() const { return m_img.rows(); }
  inline int32 planes() const { return 1; }

  inline pixel_accessor origin() const { return pixel_accessor(*this); }

  inline result_type operator()( size_t col, size_t row, size_t p=0 ) const {

    if ( m_img(col, row, p) == m_nodata_val ) return m_nodata_val;

    Vector2 lonlat = m_georef.pixel_to_lonlat(Vector2(col, row));

    // Wrap the lonlat until it is in the [0, 360) x [-90, 90) box for
    // interpolation.
    while( lonlat[0] <   0.0  ) lonlat[0] += 360.0;
    while( lonlat[0] >= 360.0 ) lonlat[0] -= 360.0;
    while( lonlat[1] <  -90.0 ) lonlat[1] += 180.0;
    while( lonlat[1] >=  90.0 ) lonlat[1] -= 180.0;

    result_type delta_height = 0.0;
    Vector2 pix = m_delta_georef.lonlat_to_pixel(lonlat);
    PixelMask<double> interp_val = m_delta(pix[0], pix[1]);
    if (is_valid(interp_val)) delta_height = interp_val;
    result_type height_above_ellipsoid = m_img(col, row, p);
    result_type height_above_geoid = height_above_ellipsoid - delta_height;

    return height_above_geoid;
  }

  /// \cond INTERNAL
  typedef AdjustDemView<typename ImageT::prerasterize_type> prerasterize_type;
  inline prerasterize_type prerasterize( BBox2i const& bbox ) const {
    return prerasterize_type( m_img.prerasterize(bbox), m_georef,
                              m_delta, m_delta_georef, m_nodata_val );
  }
  template <class DestT> inline void rasterize( DestT const& dest, BBox2i const& bbox ) const { vw::rasterize( prerasterize(bbox), dest, bbox ); }
  /// \endcond
};

template <class ImageT>
AdjustDemView<ImageT>
adjust_dem( ImageViewBase<ImageT> const& img, GeoReference const& georef,
            ImageViewRef<PixelMask<double> > const& delta,
            GeoReference const& delta_georef, double nodata_val) {
  return AdjustDemView<ImageT>( img.impl(), georef, delta, delta_georef, nodata_val );
}

struct Options : asp::BaseOptions {
  string geoid, delta_file, dem_name, output_prefix;
  double nodata_value;
  bool use_float;
};

void handle_arguments( int argc, char *argv[], Options& opt ){

  po::options_description general_options("");
  general_options.add_options()
    ("nodata_value", po::value(&opt.nodata_value)->default_value(-32767),
     "The value of no-data pixels, unless specified in the DEM.")
    ("geoid", po::value(&opt.geoid)->default_value("EGM96"), "Choose a geoid [EGM96, NAVD88].")
    ("output-prefix,o", po::value(&opt.output_prefix), "Specify the output prefix.")
    ("float", po::bool_switch(&opt.use_float)->default_value(false), "Output using float (32 bit) instead of using doubles (64 bit).");

  general_options.add( asp::BaseOptionsDescription(opt) );

  po::options_description positional("");
  positional.add_options()
    ("dem", po::value(&opt.dem_name), "Explicitly specify the DEM.");

  po::positional_options_description positional_desc;
  positional_desc.add("dem", 1);

  std::string usage("[options] <dem>");
  po::variables_map vm =
    asp::check_command_line( argc, argv, opt, general_options, general_options,
                             positional, positional_desc, usage );

  // The geoid DEM containing the adjustments
#define STR_EXPAND(tok) #tok
#define STR_QUOTE(tok) STR_EXPAND(tok)
  std::string geoid_path = STR_QUOTE(DEM_ADJUST_GEOID_PATH);
  if (opt.geoid == "EGM96"){
    opt.delta_file = geoid_path + "/" + "egm96-5.tif";
  }else if(opt.geoid == "NAVD88"){
    opt.delta_file = geoid_path + "/" + "NAVD88.tif";
  }else{
    vw_throw( ArgumentErr() << "Unknown geoid: " << opt.geoid << ".\n\n" << usage << general_options );
  }

  if ( opt.dem_name.empty() )
    vw_throw( ArgumentErr() << "Requires <dem> in order to proceed.\n\n" << usage << general_options );

  if ( opt.output_prefix.empty() ) {
    opt.output_prefix = fs::path(opt.dem_name).stem().string();
  }
}

int main( int argc, char *argv[] ) {

  // Adjust the DEM values so that they are relative to the geoid
  // rather than to the ellipsoid.
  Options opt;
  try {
    handle_arguments( argc, argv, opt );

    // Read the DEM to adjust
    DiskImageResourceGDAL dem_rsrc(opt.dem_name);
    double nodata_val = opt.nodata_value;
    if ( dem_rsrc.has_nodata_read() ) {
      nodata_val = dem_rsrc.nodata_read();
      vw_out() << "\tFound input nodata value for " << opt.dem_name << ": " << nodata_val << endl;
    }
    DiskImageView<double> dem_img(dem_rsrc);
    GeoReference dem_georef;
    read_georeference(dem_georef, dem_rsrc);

    // Read the DEM geoid containing the adjustments
    double delta_nodata_val = opt.nodata_value;
    DiskImageResourceGDAL delta_rsrc(opt.delta_file);
    if ( delta_rsrc.has_nodata_read() ) {
      delta_nodata_val = delta_rsrc.nodata_read();
      vw_out() << "\tFound input nodata value for " << opt.delta_file << ": " << delta_nodata_val << endl;
    }
    DiskImageView<double> delta_img(delta_rsrc);
    GeoReference delta_georef;
    read_georeference(delta_georef, delta_rsrc);
    ImageViewRef<PixelMask<double> > delta
      = interpolate(create_mask( delta_img, delta_nodata_val ),
                    BicubicInterpolation(), ZeroEdgeExtension());

    ImageViewRef<double> adj_dem = adjust_dem(dem_img, dem_georef,
                                              delta, delta_georef, nodata_val);

    std::string adj_dem_file = opt.output_prefix + "-adj.tif";
    vw_out() << "Writing adjusted DEM: " << adj_dem_file << std::endl;

    if ( opt.use_float ) {
      ImageViewRef<float> adj_dem_float = channel_cast<float>( adj_dem );
      boost::scoped_ptr<DiskImageResourceGDAL> rsrc( asp::build_gdal_rsrc(adj_dem_file,
                                                                          adj_dem_float, opt ) );
      rsrc->set_nodata_write( nodata_val );
      write_georeference( *rsrc, dem_georef );
      block_write_image( *rsrc, adj_dem_float,
                         TerminalProgressCallback("asp", "\t--> Applying DEM adjustment: ") );
    } else {
      boost::scoped_ptr<DiskImageResourceGDAL> rsrc( asp::build_gdal_rsrc(adj_dem_file,
                                                                          adj_dem, opt ) );
      rsrc->set_nodata_write( nodata_val );
      write_georeference( *rsrc, dem_georef );
      block_write_image( *rsrc, adj_dem,
                         TerminalProgressCallback("asp", "\t--> Applying DEM adjustment: ") );
    }

  } ASP_STANDARD_CATCHES;

  return 0;
}
<commit_msg>dem_adjust: Speed up by reading geoid in memory, other minor<commit_after>// __BEGIN_LICENSE__
//  Copyright (c) 2009-2012, United States Government as represented by the
//  Administrator of the National Aeronautics and Space Administration. All
//  rights reserved.
//
//  The NGT platform is licensed under the Apache License, Version 2.0 (the
//  "License"); you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//  http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
// __END_LICENSE__

/// \file dem_adjust.cc
///

#include <vw/FileIO.h>
#include <vw/Image.h>
#include <vw/Cartography.h>
#include <vw/Math.h>
#include <asp/Core/Macros.h>
#include <asp/Core/Common.h>

namespace po = boost::program_options;
namespace fs = boost::filesystem;

using std::endl;
using std::string;
using namespace vw;
using namespace vw::cartography;

template <class ImageT>
class DemAdjustView : public ImageViewBase<DemAdjustView<ImageT> >
{
  ImageT m_img;
  GeoReference const& m_georef;
  ImageViewRef<PixelMask<double> > const& m_delta;
  GeoReference const& m_delta_georef;
  double m_nodata_val;

public:

  typedef double pixel_type;
  typedef double result_type;
  typedef ProceduralPixelAccessor<DemAdjustView> pixel_accessor;

  DemAdjustView(ImageT const& img, GeoReference const& georef,
                ImageViewRef<PixelMask<double> > const& delta,
                GeoReference const& delta_georef, double nodata_val):
    m_img(img), m_georef(georef),
    m_delta(delta), m_delta_georef(delta_georef),
    m_nodata_val(nodata_val){}

  inline int32 cols() const { return m_img.cols(); }
  inline int32 rows() const { return m_img.rows(); }
  inline int32 planes() const { return 1; }

  inline pixel_accessor origin() const { return pixel_accessor(*this); }

  inline result_type operator()( size_t col, size_t row, size_t p=0 ) const {

    if ( m_img(col, row, p) == m_nodata_val ) return m_nodata_val;

    Vector2 lonlat = m_georef.pixel_to_lonlat(Vector2(col, row));

    // Wrap the lonlat until it is in the [0, 360) x [-90, 90) box for
    // interpolation.
    // lonlat[0] = -121; lonlat[1] = 37; // For testing (see example below)
    while( lonlat[0] <   0.0  ) lonlat[0] += 360.0;
    while( lonlat[0] >= 360.0 ) lonlat[0] -= 360.0;
    while( lonlat[1] <  -90.0 ) lonlat[1] += 180.0;
    while( lonlat[1] >=  90.0 ) lonlat[1] -= 180.0;

    result_type       delta_height           = 0.0;
    Vector2           pix                    = m_delta_georef.lonlat_to_pixel(lonlat);
    PixelMask<double> interp_val             = m_delta(pix[0], pix[1]);
    if (is_valid(interp_val)) delta_height   = interp_val;
    result_type       height_above_ellipsoid = m_img(col, row, p);
    // See the note in the main program about the formula below
    result_type height_above_geoid           = height_above_ellipsoid - delta_height;

    return height_above_geoid;
  }

  /// \cond INTERNAL
  typedef DemAdjustView<typename ImageT::prerasterize_type> prerasterize_type;
  inline prerasterize_type prerasterize( BBox2i const& bbox ) const {
    return prerasterize_type( m_img.prerasterize(bbox), m_georef,
                              m_delta, m_delta_georef, m_nodata_val );
  }
  template <class DestT> inline void rasterize( DestT const& dest, BBox2i const& bbox ) const { vw::rasterize( prerasterize(bbox), dest, bbox ); }
  /// \endcond
};

template <class ImageT>
DemAdjustView<ImageT>
dem_adjust( ImageViewBase<ImageT> const& img, GeoReference const& georef,
            ImageViewRef<PixelMask<double> > const& delta,
            GeoReference const& delta_georef, double nodata_val) {
  return DemAdjustView<ImageT>( img.impl(), georef, delta, delta_georef, nodata_val );
}

struct Options : asp::BaseOptions {
  string geoid, delta_file, dem_name, output_prefix;
  double nodata_value;
  bool use_double;
};

void handle_arguments( int argc, char *argv[], Options& opt ){

  po::options_description general_options("");
  general_options.add_options()
    ("nodata_value", po::value(&opt.nodata_value)->default_value(-32767),
     "The value of no-data pixels, unless specified in the DEM.")
    ("geoid", po::value(&opt.geoid)->default_value("EGM96"), "Choose a geoid [EGM96, NAVD88].")
    ("output-prefix,o", po::value(&opt.output_prefix), "Specify the output prefix.")
    ("double", po::bool_switch(&opt.use_double)->default_value(false)->implicit_value(true), "Output using double (64 bit) instead of float (32 bit).");

  general_options.add( asp::BaseOptionsDescription(opt) );

  po::options_description positional("");
  positional.add_options()
    ("dem", po::value(&opt.dem_name), "Explicitly specify the DEM.");

  po::positional_options_description positional_desc;
  positional_desc.add("dem", 1);

  std::string usage("[options] <dem>");
  po::variables_map vm =
    asp::check_command_line( argc, argv, opt, general_options, general_options,
                             positional, positional_desc, usage );

  // The geoid DEM containing the adjustments
#define STR_EXPAND(tok) #tok
#define STR_QUOTE(tok) STR_EXPAND(tok)
  std::string geoid_path = STR_QUOTE(DEM_ADJUST_GEOID_PATH);
  if (opt.geoid == "EGM96"){
    opt.delta_file = geoid_path + "/" + "egm96-5.tif";
  }else if(opt.geoid == "NAVD88"){
    opt.delta_file = geoid_path + "/" + "NAVD88.tif";
  }else{
    vw_throw( ArgumentErr() << "Unknown geoid: " << opt.geoid << ".\n\n" << usage << general_options );
  }

  if ( opt.dem_name.empty() )
    vw_throw( ArgumentErr() << "Requires <dem> in order to proceed.\n\n" << usage << general_options );

  if ( opt.output_prefix.empty() ) {
    opt.output_prefix = fs::path(opt.dem_name).stem().string();
  }
}

// Given a DEM, with each height value relative to the datum
// ellipsoid, convert the heights to be relative to the geoid.

// From: http://earth-info.nga.mil/GandG/wgs84/gravitymod/egm96/intpthel.html
// Geoid heights can be used to convert between orthometric
// heights (approximately mean sea level) and ellipsoid heights
// according to the formula:
// h   =   H   +   N
// where,
// h = WGS 84 Ellipsoid height
// H = Orthometric height
// N = EGM96 Geoid height
// Therefore: H = h - N.

// We support two geoids: EGM96 and  NAVD88.
// Online tools for verification:
// EGM96:  http://earth-info.nga.mil/GandG/wgs84/gravitymod/egm96/intpt.html
// NAVD88: http://www.ngs.noaa.gov/cgi-bin/GEOID_STUFF/geoid09_prompt1.prl
//
// Example, at lat = 37 and lon = -121 (Basalt Hills, CA), we have
// EGM96  geoid height = -32.69
// NAVD88 geoid height = -32.931

int main( int argc, char *argv[] ) {

  Options opt;
  try {
    handle_arguments( argc, argv, opt );

    // Read the DEM to adjust
    DiskImageResourceGDAL dem_rsrc(opt.dem_name);
    double nodata_val = opt.nodata_value;
    if ( dem_rsrc.has_nodata_read() ) {
      nodata_val = dem_rsrc.nodata_read();
      vw_out() << "\tFound input nodata value for " << opt.dem_name << ": " << nodata_val << endl;
    }
    DiskImageView<double> dem_img(dem_rsrc);
    GeoReference dem_georef;
    read_georeference(dem_georef, dem_rsrc);

    // Read the geoid containing the adjustments. Read it in memory
    // entirely to dramatically speed up the computations.
    double delta_nodata_val = std::numeric_limits<float>::quiet_NaN();
    DiskImageResourceGDAL delta_rsrc(opt.delta_file);
    if ( delta_rsrc.has_nodata_read() ) {
      delta_nodata_val = delta_rsrc.nodata_read();
    }
    vw_out() << "\tAdjusting the DEM using the geoid: " << opt.delta_file << endl;
    ImageView<float> delta_img = DiskImageView<float>(delta_rsrc);
    GeoReference delta_georef;
    read_georeference(delta_georef, delta_rsrc);
    ImageViewRef<PixelMask<double> > delta
      = interpolate(create_mask( pixel_cast<double>(delta_img), delta_nodata_val ),
                    BicubicInterpolation(), ZeroEdgeExtension());

    ImageViewRef<double> adj_dem = dem_adjust(dem_img, dem_georef,
                                              delta, delta_georef, nodata_val);

    std::string adj_dem_file = opt.output_prefix + "-adj.tif";
    vw_out() << "Writing adjusted DEM: " << adj_dem_file << std::endl;

    if ( opt.use_double ) {
      // Output as double
      boost::scoped_ptr<DiskImageResourceGDAL> rsrc( asp::build_gdal_rsrc(adj_dem_file,
                                                                          adj_dem, opt ) );
      rsrc->set_nodata_write( nodata_val );
      write_georeference( *rsrc, dem_georef );
      block_write_image( *rsrc, adj_dem,
                         TerminalProgressCallback("asp", "\t--> Applying DEM adjustment: ") );
    }else{
      // Output as float
      ImageViewRef<float> adj_dem_float = channel_cast<float>( adj_dem );
      boost::scoped_ptr<DiskImageResourceGDAL> rsrc( asp::build_gdal_rsrc(adj_dem_file,
                                                                          adj_dem_float, opt ) );
      rsrc->set_nodata_write( nodata_val );
      write_georeference( *rsrc, dem_georef );
      block_write_image( *rsrc, adj_dem_float,
                         TerminalProgressCallback("asp", "\t--> Applying DEM adjustment: ") );
    }

  } ASP_STANDARD_CATCHES;

  return 0;
}
<|endoftext|>


<commit_before>/******************************************************************************
The MIT License(MIT)

Embedded Template Library.
https://github.com/ETLCPP/etl
https://www.etlcpp.com

Copyright(c) 2014 jwellbelove

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files(the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and / or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions :

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
******************************************************************************/

#include "unit_test_framework.h"

#include "etl/instance_count.h"

#include <list>
#include <vector>
#include <numeric>

namespace
{
  SUITE(test_instance_count)
  {
    //*************************************************************************
    TEST(test_count)
    {
      struct Test1 : public etl::instance_count<Test1>
      {};

      struct Test2 : public etl::instance_count<Test2>
      {};

      CHECK_EQUAL(0, Test1::get_instance_count());
      CHECK_EQUAL(0, Test2::get_instance_count());

      Test1 test1a;
      CHECK_EQUAL(1, Test1::get_instance_count());
      CHECK_EQUAL(0, Test2::get_instance_count());

      Test1 test1b;
      Test2 test2a;
      CHECK_EQUAL(2, Test1::get_instance_count());
      CHECK_EQUAL(1, Test2::get_instance_count());

      Test2* ptest2b = new Test2;
      CHECK_EQUAL(2, Test1::get_instance_count());
      CHECK_EQUAL(2, Test2::get_instance_count());

      Test2 test2c(test2a);
      CHECK_EQUAL(2, Test1::get_instance_count());
      CHECK_EQUAL(3, Test2::get_instance_count());

      delete ptest2b;
      CHECK_EQUAL(2, Test1::get_instance_count());
      CHECK_EQUAL(2, Test2::get_instance_count());
    }
  };
}
<commit_msg>Added optional counter type to instance_count.<commit_after>/******************************************************************************
The MIT License(MIT)

Embedded Template Library.
https://github.com/ETLCPP/etl
https://www.etlcpp.com

Copyright(c) 2014 jwellbelove

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files(the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and / or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions :

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
******************************************************************************/

#include "unit_test_framework.h"

#include "etl/instance_count.h"

#include <list>
#include <vector>
#include <numeric>
#include <atomic>

namespace
{
  SUITE(test_instance_count)
  {
    //*************************************************************************
    TEST(test_count)
    {
      struct Test1 : public etl::instance_count<Test1>
      {};

      struct Test2 : public etl::instance_count<Test2>
      {};

      CHECK_EQUAL(0, Test1::get_instance_count());
      CHECK_EQUAL(0, Test2::get_instance_count());

      Test1 test1a;
      CHECK_EQUAL(1, Test1::get_instance_count());
      CHECK_EQUAL(0, Test2::get_instance_count());

      Test1 test1b;
      Test2 test2a;
      CHECK_EQUAL(2, Test1::get_instance_count());
      CHECK_EQUAL(1, Test2::get_instance_count());

      Test2* ptest2b = new Test2;
      CHECK_EQUAL(2, Test1::get_instance_count());
      CHECK_EQUAL(2, Test2::get_instance_count());

      Test2 test2c(test2a);
      CHECK_EQUAL(2, Test1::get_instance_count());
      CHECK_EQUAL(3, Test2::get_instance_count());

      delete ptest2b;
      CHECK_EQUAL(2, Test1::get_instance_count());
      CHECK_EQUAL(2, Test2::get_instance_count());
    }

    //*************************************************************************
    TEST(test_atomic_count)
    {
      struct Test1 : public etl::instance_count<Test1, std::atomic_uint8_t>
      {};

      struct Test2 : public etl::instance_count<Test2>
      {};

      CHECK_EQUAL(0, Test1::get_instance_count());
      CHECK_EQUAL(0, Test2::get_instance_count());

      Test1 test1a;
      CHECK_EQUAL(1, Test1::get_instance_count());
      CHECK_EQUAL(0, Test2::get_instance_count());

      Test1 test1b;
      Test2 test2a;
      CHECK_EQUAL(2, Test1::get_instance_count());
      CHECK_EQUAL(1, Test2::get_instance_count());

      Test2* ptest2b = new Test2;
      CHECK_EQUAL(2, Test1::get_instance_count());
      CHECK_EQUAL(2, Test2::get_instance_count());

      Test2 test2c(test2a);
      CHECK_EQUAL(2, Test1::get_instance_count());
      CHECK_EQUAL(3, Test2::get_instance_count());

      delete ptest2b;
      CHECK_EQUAL(2, Test1::get_instance_count());
      CHECK_EQUAL(2, Test2::get_instance_count());
    }
  };
}
<|endoftext|>


<commit_before>//
// Created by Davide  Caroselli on 28/09/16.
//

#include "SuffixArray.h"
#include "dbkv.h"
#include <rocksdb/slice_transform.h>
#include <rocksdb/merge_operator.h>
#include <thread>
#include <boost/filesystem.hpp>
#include <iostream>
#include <util/hashutils.h>

namespace fs = boost::filesystem;

using namespace rocksdb;
using namespace mmt;
using namespace mmt::sapt;

const domain_t mmt::sapt::kBackgroundModelDomain = 0;
static const string kGlobalInfoKey = MakeEmptyKey(kGlobalInfoKeyType);

/*
 * MergePositionOperator
 */

namespace mmt {
    namespace sapt {

        class MergePositionOperator : public AssociativeMergeOperator {
        public:
            virtual bool Merge(const Slice &key, const Slice *existing_value, const Slice &value, string *new_value,
                               Logger *logger) const override {
                switch (key.data_[0]) {
                    case kSourcePrefixKeyType:
                    case kTargetPrefixKeyType:
                        MergePositionLists(existing_value, value, new_value);
                        return true;
                    default:
                        return false;
                }
            }

            inline void MergePositionLists(const Slice *existing_value, const Slice &value, string *new_value) const {
                if (existing_value)
                    *new_value = existing_value->ToString() + value.ToString();
                else
                    *new_value = value.ToString();
            }

            virtual const char *Name() const override {
                return "MergePositionOperator";
            }
        };

    }
}

/*
 * SuffixArray - Initialization
 */

SuffixArray::SuffixArray(const string &modelPath, uint8_t prefixLength,
                         bool prepareForBulkLoad) throw(index_exception, storage_exception) :
        prefixLength(prefixLength) {
    fs::path modelDir(modelPath);

    if (!fs::is_directory(modelDir))
        throw invalid_argument("Invalid model path: " + modelPath);

    fs::path storageFile = fs::absolute(modelDir / fs::path("corpora.bin"));
    fs::path indexPath = fs::absolute(modelDir / fs::path("index"));

    rocksdb::Options options;
    options.create_if_missing = true;
    options.merge_operator.reset(new MergePositionOperator);
    options.max_open_files = -1;
    options.compaction_style = kCompactionStyleLevel;

    if (prepareForBulkLoad) {
        options.PrepareForBulkLoad();
    } else {
        unsigned cpus = thread::hardware_concurrency();

        if (cpus > 1)
            options.IncreaseParallelism(cpus > 4 ? 4 : 2);

        options.level0_file_num_compaction_trigger = 8;
        options.level0_slowdown_writes_trigger = 17;
        options.level0_stop_writes_trigger = 24;
        options.num_levels = 4;

        options.write_buffer_size = 64L * 1024L * 1024L;
        options.max_write_buffer_number = 3;
        options.target_file_size_base = 64L * 1024L * 1024L;
        options.max_bytes_for_level_base = 512L * 1024L * 1024L;
        options.max_bytes_for_level_multiplier = 8;
    }

    Status status = DB::Open(options, indexPath.string(), &db);
    if (!status.ok())
        throw index_exception(status.ToString());

    db->CompactRange(CompactRangeOptions(), NULL, NULL);

    // Read streams
    string raw_streams;
    int64_t storageSize = 0;

    db->Get(ReadOptions(), kGlobalInfoKey, &raw_streams);
    DeserializeGlobalInfo(raw_streams.data(), raw_streams.size(), &storageSize, &streams);

    // Load storage
    storage = new CorpusStorage(storageFile.string(), storageSize);
}

SuffixArray::~SuffixArray() {
    delete db;
    delete storage;
}

/*
 * SuffixArray - Indexing
 */

void SuffixArray::ForceCompaction() {
    db->CompactRange(CompactRangeOptions(), NULL, NULL);
}

void SuffixArray::PutBatch(UpdateBatch &batch) throw(index_exception, storage_exception) {
    WriteBatch writeBatch;

    // Compute prefixes
    unordered_map<string, PostingList> sourcePrefixes;
    unordered_map<string, PostingList> targetPrefixes;

    for (auto entry = batch.data.begin(); entry != batch.data.end(); ++entry) {
        domain_t domain = entry->domain;

        int64_t offset = storage->Append(entry->source, entry->target, entry->alignment);
        AddPrefixesToBatch(true, domain, entry->source, offset, sourcePrefixes);
        AddPrefixesToBatch(false, kBackgroundModelDomain, entry->target, offset, targetPrefixes);
    }

    int64_t storageSize = storage->Flush();

    // Add prefixes to write batch
    for (auto prefix = sourcePrefixes.begin(); prefix != sourcePrefixes.end(); ++prefix) {
        string value = prefix->second.Serialize();
        writeBatch.Merge(prefix->first, value);
    }
    for (auto prefix = targetPrefixes.begin(); prefix != targetPrefixes.end(); ++prefix) {
        string value = prefix->second.Serialize();
        writeBatch.Merge(prefix->first, value);
    }

    // Write global info
    writeBatch.Put(kGlobalInfoKey, SerializeGlobalInfo(batch.streams, storageSize));

    // Commit write batch
    Status status = db->Write(WriteOptions(), &writeBatch);
    if (!status.ok())
        throw index_exception("Unable to write to index: " + status.ToString());

    // Reset streams and domains
    streams = batch.GetStreams();
}

void SuffixArray::AddPrefixesToBatch(bool isSource, domain_t domain, const vector<wid_t> &sentence,
                                     int64_t location, unordered_map<string, PostingList> &outBatch) {
    for (length_t start = 0; start < sentence.size(); ++start) {
        size_t length = prefixLength;
        if (start + length > sentence.size())
            length = sentence.size() - start;

        // Add to background model
        string key = MakePrefixKey(isSource, kBackgroundModelDomain, sentence, start, length);
        outBatch[key].Append(kBackgroundModelDomain, location, start);

        // Add to domain
        if (domain != kBackgroundModelDomain) {
            string dkey = MakePrefixKey(isSource, domain, sentence, start, length);
            outBatch[dkey].Append(domain, location, start);
        }
    }
}

/*
 * SuffixArray - Query
 */

size_t SuffixArray::CountOccurrences(bool isSource, const vector<wid_t> &phrase) {
    PostingList locations(phrase);
    CollectLocations(isSource, kBackgroundModelDomain, phrase, locations);

    return locations.size();
}

void SuffixArray::GetRandomSamples(const vector<wid_t> &phrase, size_t limit, vector<sample_t> &outSamples,
                                   const context_t *context, bool searchInBackground) {
    PostingList inContextLocations(phrase);
    PostingList outContextLocations(phrase);
    size_t remaining = limit;

    if (context) {
        for (auto score = context->begin(); score != context->end(); ++score) {
            CollectPositions(true, score->domain, phrase, inContextPositions);

            std::cerr << "Found " << outSamples.size()  << " samples" << std::endl;

            if (limit > 0) {
                if (inContextLocations.size() >= limit) {
                    remaining = 0;
                    break;
                } else {
                    remaining = limit - inContextLocations.size();
                }
            }
        }
    }

    if (searchInBackground && (limit == 0 || remaining > 0)) {
        unordered_set<int64_t> coveredLocations = inContextLocations.GetLocations();
        CollectLocations(true, kBackgroundModelDomain, phrase, outContextLocations, &coveredLocations);
    }

    outSamples.clear();

    ssize_t inContextSize = inContextLocations.size() - limit;
    if (inContextSize < 0) {
        map<int64_t, pair<domain_t, vector<length_t>>> inContext = inContextLocations.GetSamples();
        map<int64_t, pair<domain_t, vector<length_t>>> outContext =
                outContextLocations.GetSamples((size_t) -inContextSize, words_hash(phrase));

        Retrieve(inContext, outSamples);
        Retrieve(outContext, outSamples);
    } else {
        map<int64_t, pair<domain_t, vector<length_t>>> inContext = inContextLocations.GetSamples(limit);
        Retrieve(inContext, outSamples);
    }
}

void SuffixArray::CollectLocations(bool isSource, domain_t domain, const vector<wid_t> &sentence,
                                   PostingList &output, unordered_set<int64_t> *coveredLocations) {
    length_t sentenceLength = (length_t) sentence.size();

    if (sentenceLength <= prefixLength) {
        CollectLocations(isSource, domain, sentence, 0, sentence.size(), output, coveredLocations);
    } else {
        length_t start = 0;
        PostingList collected(sentence);

        while (start < sentenceLength) {
            if (start + prefixLength > sentenceLength)
                start = sentenceLength - prefixLength;

            if (start == 0) {
                CollectLocations(isSource, domain, sentence, start, prefixLength, collected, coveredLocations);
            } else {
                PostingList successors(sentence, start, prefixLength);
                CollectLocations(isSource, domain, sentence, start, prefixLength, successors, coveredLocations);

                collected.Retain(successors, start);
            }

            if (collected.empty())
                break;

            start += prefixLength;
        }

        output.Append(collected);
    }
}

void SuffixArray::CollectLocations(bool isSource, domain_t domain, const vector<wid_t> &phrase,
                                   size_t offset, size_t length, PostingList &output,
                                   const unordered_set<int64_t> *coveredLocations) {
    string key = MakePrefixKey(isSource, domain, phrase, offset, length);

    if (length == prefixLength) {
        string value;
        db->Get(ReadOptions(), key, &value);

        output.Append(domain, value.data(), value.size(), coveredLocations);
    } else {
        Iterator *it = db->NewIterator(ReadOptions());

        for (it->Seek(key); it->Valid() && it->key().starts_with(key); it->Next()) {
            Slice value = it->value();
            output.Append(domain, value.data_, value.size_, coveredLocations);
        }

        delete it;
    }
}

void
SuffixArray::Retrieve(const map<int64_t, pair<domain_t, vector<length_t>>> &locations, vector<sample_t> &outSamples) {
    // Resolve positions
    outSamples.reserve(outSamples.size() + locations.size());

    for (auto location = locations.begin(); location != locations.end(); ++location) {
        auto &value = location->second;

        sample_t sample;
        sample.domain = value.first;
        sample.offsets = value.second;

        storage->Retrieve(location->first, &sample.source, &sample.target, &sample.alignment);

        outSamples.push_back(sample);
    }
}
<commit_msg>fixed name mismatch after merging<commit_after>//
// Created by Davide  Caroselli on 28/09/16.
//

#include "SuffixArray.h"
#include "dbkv.h"
#include <rocksdb/slice_transform.h>
#include <rocksdb/merge_operator.h>
#include <thread>
#include <boost/filesystem.hpp>
#include <iostream>
#include <util/hashutils.h>

namespace fs = boost::filesystem;

using namespace rocksdb;
using namespace mmt;
using namespace mmt::sapt;

const domain_t mmt::sapt::kBackgroundModelDomain = 0;
static const string kGlobalInfoKey = MakeEmptyKey(kGlobalInfoKeyType);

/*
 * MergePositionOperator
 */

namespace mmt {
    namespace sapt {

        class MergePositionOperator : public AssociativeMergeOperator {
        public:
            virtual bool Merge(const Slice &key, const Slice *existing_value, const Slice &value, string *new_value,
                               Logger *logger) const override {
                switch (key.data_[0]) {
                    case kSourcePrefixKeyType:
                    case kTargetPrefixKeyType:
                        MergePositionLists(existing_value, value, new_value);
                        return true;
                    default:
                        return false;
                }
            }

            inline void MergePositionLists(const Slice *existing_value, const Slice &value, string *new_value) const {
                if (existing_value)
                    *new_value = existing_value->ToString() + value.ToString();
                else
                    *new_value = value.ToString();
            }

            virtual const char *Name() const override {
                return "MergePositionOperator";
            }
        };

    }
}

/*
 * SuffixArray - Initialization
 */

SuffixArray::SuffixArray(const string &modelPath, uint8_t prefixLength,
                         bool prepareForBulkLoad) throw(index_exception, storage_exception) :
        prefixLength(prefixLength) {
    fs::path modelDir(modelPath);

    if (!fs::is_directory(modelDir))
        throw invalid_argument("Invalid model path: " + modelPath);

    fs::path storageFile = fs::absolute(modelDir / fs::path("corpora.bin"));
    fs::path indexPath = fs::absolute(modelDir / fs::path("index"));

    rocksdb::Options options;
    options.create_if_missing = true;
    options.merge_operator.reset(new MergePositionOperator);
    options.max_open_files = -1;
    options.compaction_style = kCompactionStyleLevel;

    if (prepareForBulkLoad) {
        options.PrepareForBulkLoad();
    } else {
        unsigned cpus = thread::hardware_concurrency();

        if (cpus > 1)
            options.IncreaseParallelism(cpus > 4 ? 4 : 2);

        options.level0_file_num_compaction_trigger = 8;
        options.level0_slowdown_writes_trigger = 17;
        options.level0_stop_writes_trigger = 24;
        options.num_levels = 4;

        options.write_buffer_size = 64L * 1024L * 1024L;
        options.max_write_buffer_number = 3;
        options.target_file_size_base = 64L * 1024L * 1024L;
        options.max_bytes_for_level_base = 512L * 1024L * 1024L;
        options.max_bytes_for_level_multiplier = 8;
    }

    Status status = DB::Open(options, indexPath.string(), &db);
    if (!status.ok())
        throw index_exception(status.ToString());

    db->CompactRange(CompactRangeOptions(), NULL, NULL);

    // Read streams
    string raw_streams;
    int64_t storageSize = 0;

    db->Get(ReadOptions(), kGlobalInfoKey, &raw_streams);
    DeserializeGlobalInfo(raw_streams.data(), raw_streams.size(), &storageSize, &streams);

    // Load storage
    storage = new CorpusStorage(storageFile.string(), storageSize);
}

SuffixArray::~SuffixArray() {
    delete db;
    delete storage;
}

/*
 * SuffixArray - Indexing
 */

void SuffixArray::ForceCompaction() {
    db->CompactRange(CompactRangeOptions(), NULL, NULL);
}

void SuffixArray::PutBatch(UpdateBatch &batch) throw(index_exception, storage_exception) {
    WriteBatch writeBatch;

    // Compute prefixes
    unordered_map<string, PostingList> sourcePrefixes;
    unordered_map<string, PostingList> targetPrefixes;

    for (auto entry = batch.data.begin(); entry != batch.data.end(); ++entry) {
        domain_t domain = entry->domain;

        int64_t offset = storage->Append(entry->source, entry->target, entry->alignment);
        AddPrefixesToBatch(true, domain, entry->source, offset, sourcePrefixes);
        AddPrefixesToBatch(false, kBackgroundModelDomain, entry->target, offset, targetPrefixes);
    }

    int64_t storageSize = storage->Flush();

    // Add prefixes to write batch
    for (auto prefix = sourcePrefixes.begin(); prefix != sourcePrefixes.end(); ++prefix) {
        string value = prefix->second.Serialize();
        writeBatch.Merge(prefix->first, value);
    }
    for (auto prefix = targetPrefixes.begin(); prefix != targetPrefixes.end(); ++prefix) {
        string value = prefix->second.Serialize();
        writeBatch.Merge(prefix->first, value);
    }

    // Write global info
    writeBatch.Put(kGlobalInfoKey, SerializeGlobalInfo(batch.streams, storageSize));

    // Commit write batch
    Status status = db->Write(WriteOptions(), &writeBatch);
    if (!status.ok())
        throw index_exception("Unable to write to index: " + status.ToString());

    // Reset streams and domains
    streams = batch.GetStreams();
}

void SuffixArray::AddPrefixesToBatch(bool isSource, domain_t domain, const vector<wid_t> &sentence,
                                     int64_t location, unordered_map<string, PostingList> &outBatch) {
    for (length_t start = 0; start < sentence.size(); ++start) {
        size_t length = prefixLength;
        if (start + length > sentence.size())
            length = sentence.size() - start;

        // Add to background model
        string key = MakePrefixKey(isSource, kBackgroundModelDomain, sentence, start, length);
        outBatch[key].Append(kBackgroundModelDomain, location, start);

        // Add to domain
        if (domain != kBackgroundModelDomain) {
            string dkey = MakePrefixKey(isSource, domain, sentence, start, length);
            outBatch[dkey].Append(domain, location, start);
        }
    }
}

/*
 * SuffixArray - Query
 */

size_t SuffixArray::CountOccurrences(bool isSource, const vector<wid_t> &phrase) {
    PostingList locations(phrase);
    CollectLocations(isSource, kBackgroundModelDomain, phrase, locations);

    return locations.size();
}

void SuffixArray::GetRandomSamples(const vector<wid_t> &phrase, size_t limit, vector<sample_t> &outSamples,
                                   const context_t *context, bool searchInBackground) {
    PostingList inContextLocations(phrase);
    PostingList outContextLocations(phrase);
    size_t remaining = limit;

    if (context) {
        for (auto score = context->begin(); score != context->end(); ++score) {
            CollectLocations(true, score->domain, phrase, inContextLocations);

            std::cerr << "Found " << outSamples.size()  << " samples" << std::endl;

            if (limit > 0) {
                if (inContextLocations.size() >= limit) {
                    remaining = 0;
                    break;
                } else {
                    remaining = limit - inContextLocations.size();
                }
            }
        }
    }

    if (searchInBackground && (limit == 0 || remaining > 0)) {
        unordered_set<int64_t> coveredLocations = inContextLocations.GetLocations();
        CollectLocations(true, kBackgroundModelDomain, phrase, outContextLocations, &coveredLocations);
    }

    outSamples.clear();

    ssize_t inContextSize = inContextLocations.size() - limit;
    if (inContextSize < 0) {
        map<int64_t, pair<domain_t, vector<length_t>>> inContext = inContextLocations.GetSamples();
        map<int64_t, pair<domain_t, vector<length_t>>> outContext =
                outContextLocations.GetSamples((size_t) -inContextSize, words_hash(phrase));

        Retrieve(inContext, outSamples);
        Retrieve(outContext, outSamples);
    } else {
        map<int64_t, pair<domain_t, vector<length_t>>> inContext = inContextLocations.GetSamples(limit);
        Retrieve(inContext, outSamples);
    }
}

void SuffixArray::CollectLocations(bool isSource, domain_t domain, const vector<wid_t> &sentence,
                                   PostingList &output, unordered_set<int64_t> *coveredLocations) {
    length_t sentenceLength = (length_t) sentence.size();

    if (sentenceLength <= prefixLength) {
        CollectLocations(isSource, domain, sentence, 0, sentence.size(), output, coveredLocations);
    } else {
        length_t start = 0;
        PostingList collected(sentence);

        while (start < sentenceLength) {
            if (start + prefixLength > sentenceLength)
                start = sentenceLength - prefixLength;

            if (start == 0) {
                CollectLocations(isSource, domain, sentence, start, prefixLength, collected, coveredLocations);
            } else {
                PostingList successors(sentence, start, prefixLength);
                CollectLocations(isSource, domain, sentence, start, prefixLength, successors, coveredLocations);

                collected.Retain(successors, start);
            }

            if (collected.empty())
                break;

            start += prefixLength;
        }

        output.Append(collected);
    }
}

void SuffixArray::CollectLocations(bool isSource, domain_t domain, const vector<wid_t> &phrase,
                                   size_t offset, size_t length, PostingList &output,
                                   const unordered_set<int64_t> *coveredLocations) {
    string key = MakePrefixKey(isSource, domain, phrase, offset, length);

    if (length == prefixLength) {
        string value;
        db->Get(ReadOptions(), key, &value);

        output.Append(domain, value.data(), value.size(), coveredLocations);
    } else {
        Iterator *it = db->NewIterator(ReadOptions());

        for (it->Seek(key); it->Valid() && it->key().starts_with(key); it->Next()) {
            Slice value = it->value();
            output.Append(domain, value.data_, value.size_, coveredLocations);
        }

        delete it;
    }
}

void
SuffixArray::Retrieve(const map<int64_t, pair<domain_t, vector<length_t>>> &locations, vector<sample_t> &outSamples) {
    // Resolve positions
    outSamples.reserve(outSamples.size() + locations.size());

    for (auto location = locations.begin(); location != locations.end(); ++location) {
        auto &value = location->second;

        sample_t sample;
        sample.domain = value.first;
        sample.offsets = value.second;

        storage->Retrieve(location->first, &sample.source, &sample.target, &sample.alignment);

        outSamples.push_back(sample);
    }
}
<|endoftext|>


<commit_before>/*=========================================================================

 Program:   ORFEO Toolbox
 Language:  C++
 Date:      $Date$
 Version:   $Revision$

 Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
 See OTBCopyright.txt for details.

 This software is distributed WITHOUT ANY WARRANTY; without even
 the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 PURPOSE.  See the above copyright notices for more information.

 =========================================================================*/
#include "otbWrapperApplication.h"
#include "otbWrapperApplicationFactory.h"

#include "otbStatisticsXMLFileWriter.h"
#include "otbStreamingStatisticsVectorImageFilter.h"

namespace otb
{
namespace Wrapper
{

class EstimateImagesStatistics: public Application
{
public:
  /** Standard class typedefs. */
  typedef EstimateImagesStatistics Self;
  typedef Application Superclass;
  typedef itk::SmartPointer<Self> Pointer;
  typedef itk::SmartPointer<const Self> ConstPointer;

  /** Standard macro */
  itkNewMacro(Self);

  itkTypeMacro(EstimateImagesStatistics, otb::Application);

private:
  EstimateImagesStatistics()
  {
    SetName("EstimateImagesStatistics");
    SetDescription("Estimates mean/standard deviation for all images in the input list and optionally saves the results in an XML file");

    SetDocName("Estimate Image Statistics Application");
    SetDocLongDescription("This application estimates mean/standard deviation for all images in the input list and optionally saves the results in an XML file.");
    SetDocLimitations("None");
    SetDocAuthors("OTB-Team");
    SetDocSeeAlso(" ");
    SetDocCLExample("otbApplicationLauncherCommandLine EstimateImagesStatistics ${OTB-BIN}/bin  --il ${OTB-Data}/Input/Classification/QB_1_ortho.tif ${OTB-Data}/Input/Classification/QB_2_ortho.tif ${OTB-Data}/Input/Classification/QB_3_ortho.tif --out EstimateImageStatisticsQB123.xml");
    AddDocTag("Classification");
  }

  virtual ~EstimateImagesStatistics()
  {
  }

  void DoCreateParameters()
  {
    AddParameter(ParameterType_InputImageList, "il", "Input Image List ");
    SetParameterDescription( "il", "Input Image List filename." );

    AddParameter(ParameterType_Filename, "out", "Output XML file ");
    SetParameterDescription( "out", "Name of the XML file where the statistics are saved for future reuse" );
    MandatoryOff("out");
  }

  void DoUpdateParameters()
  {
    // Nothing to do here : all parameters are independent
  }

  void DoExecute()
  {
    //Statistics estimator
    typedef otb::StreamingStatisticsVectorImageFilter<FloatVectorImageType> StreamingStatisticsVImageFilterType;

    // Samples
    typedef double ValueType;
    typedef itk::VariableLengthVector<ValueType> MeasurementType;

    unsigned int nbSamples = 0;
    unsigned int nbBands = 0;

    // Build a Measurement Vector of mean
    MeasurementType mean;

    // Build a MeasurementVector of variance
    MeasurementType variance;

    FloatVectorImageListType* imageList = GetParameterImageList("il");

    //Iterate over all input images
    for (unsigned int imageId = 0; imageId < imageList->Size(); ++imageId)
      {
      FloatVectorImageType* image = imageList->GetNthElement(imageId);

      if (nbBands == 0)
        {
        nbBands = image->GetNumberOfComponentsPerPixel();
        }
      else if (nbBands != image->GetNumberOfComponentsPerPixel())
        {
        itkExceptionMacro(<< "The image #" << imageId << " has " << image->GetNumberOfComponentsPerPixel()
            << " bands, while the first one has " << nbBands );
        }

      FloatVectorImageType::SizeType size = image->GetLargestPossibleRegion().GetSize();

      //Set the measurement vectors size if it's the first iteration
      if (imageId == 0)
        {
        mean.SetSize(nbBands);
        mean.Fill(0.);
        variance.SetSize(nbBands);
        variance.Fill(0.);
        }

      // Compute Statistics of each VectorImage
      StreamingStatisticsVImageFilterType::Pointer statsEstimator = StreamingStatisticsVImageFilterType::New();
      statsEstimator->SetInput(image);
      statsEstimator->Update();
      mean += statsEstimator->GetMean();
      for (unsigned int itBand = 0; itBand < nbBands; itBand++)
        {
        variance[itBand] += (size[0] * size[1] - 1) * (statsEstimator->GetCovariance())(itBand, itBand);
        }
      //Increment nbSamples
      nbSamples += size[0] * size[1] * nbBands;
      }

    //Divide by the number of input images to get the mean over all layers
    mean /= imageList->Size();
    //Apply the pooled variance formula
    variance /= (nbSamples - imageList->Size());

    MeasurementType stddev;
    stddev.SetSize(nbBands);
    stddev.Fill(0.);
    for (unsigned int i = 0; i < variance.GetSize(); ++i)
      {
      stddev[i] = vcl_sqrt(variance[i]);
      }

    if( HasValue( "out" )==true )
      {
      // Write the Statistics via the statistic writer
      typedef otb::StatisticsXMLFileWriter<MeasurementType> StatisticsWriter;
      StatisticsWriter::Pointer writer = StatisticsWriter::New();
      writer->SetFileName(GetParameterString("out"));
      writer->AddInput("mean", mean);
      writer->AddInput("stddev", stddev);
      writer->Update();
      }
    else
      {
      std::cout<<"Mean: "<<mean<<std::endl;
      std::cout<<"Standard Deviation: "<<stddev<<std::endl;
      }
  }

  itk::LightObject::Pointer m_FilterRef;
};

}
}

OTB_APPLICATION_EXPORT(otb::Wrapper::EstimateImagesStatistics)
<commit_msg>DOC: doc update.<commit_after>/*=========================================================================

 Program:   ORFEO Toolbox
 Language:  C++
 Date:      $Date$
 Version:   $Revision$

 Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
 See OTBCopyright.txt for details.

 This software is distributed WITHOUT ANY WARRANTY; without even
 the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 PURPOSE.  See the above copyright notices for more information.

 =========================================================================*/
#include "otbWrapperApplication.h"
#include "otbWrapperApplicationFactory.h"

#include "otbStatisticsXMLFileWriter.h"
#include "otbStreamingStatisticsVectorImageFilter.h"

namespace otb
{
namespace Wrapper
{

class EstimateImagesStatistics: public Application
{
public:
  /** Standard class typedefs. */
  typedef EstimateImagesStatistics Self;
  typedef Application Superclass;
  typedef itk::SmartPointer<Self> Pointer;
  typedef itk::SmartPointer<const Self> ConstPointer;

  /** Standard macro */
  itkNewMacro(Self);

  itkTypeMacro(EstimateImagesStatistics, otb::Application);

private:
  EstimateImagesStatistics()
  {
    SetName("EstimateImagesStatistics");
    SetDescription("Estimates mean/standard deviation for all images in the input list and optionally saves the results in an XML file");

    SetDocName("Estimate Image Statistics Application");
    SetDocLongDescription("This application estimates mean/standard deviation for all images in the input list and optionally saves the results in an XML file.");
    SetDocLimitations("None");
    SetDocAuthors("OTB-Team");
    SetDocSeeAlso(" ");
    SetDocCLExample("otbApplicationLauncherCommandLine EstimateImagesStatistics ${OTB-BIN}/bin  --il ${OTB-Data}/Input/Classification/QB_1_ortho.tif ${OTB-Data}/Input/Classification/QB_2_ortho.tif ${OTB-Data}/Input/Classification/QB_3_ortho.tif --out EstimateImageStatisticsQB123.xml");
    AddDocTag("Classification");
  }

  virtual ~EstimateImagesStatistics()
  {
  }

  void DoCreateParameters()
  {
    AddParameter(ParameterType_InputImageList, "il", "Input Image List");
    SetParameterDescription( "il", "Input Image List filename." );

    AddParameter(ParameterType_Filename, "out", "Output XML file");
    SetParameterDescription( "out", "Name of the XML file where the statistics are saved for future reuse" );
    MandatoryOff("out");
  }

  void DoUpdateParameters()
  {
    // Nothing to do here : all parameters are independent
  }

  void DoExecute()
  {
    //Statistics estimator
    typedef otb::StreamingStatisticsVectorImageFilter<FloatVectorImageType> StreamingStatisticsVImageFilterType;

    // Samples
    typedef double ValueType;
    typedef itk::VariableLengthVector<ValueType> MeasurementType;

    unsigned int nbSamples = 0;
    unsigned int nbBands = 0;

    // Build a Measurement Vector of mean
    MeasurementType mean;

    // Build a MeasurementVector of variance
    MeasurementType variance;

    FloatVectorImageListType* imageList = GetParameterImageList("il");

    //Iterate over all input images
    for (unsigned int imageId = 0; imageId < imageList->Size(); ++imageId)
      {
      FloatVectorImageType* image = imageList->GetNthElement(imageId);

      if (nbBands == 0)
        {
        nbBands = image->GetNumberOfComponentsPerPixel();
        }
      else if (nbBands != image->GetNumberOfComponentsPerPixel())
        {
        itkExceptionMacro(<< "The image #" << imageId << " has " << image->GetNumberOfComponentsPerPixel()
            << " bands, while the first one has " << nbBands );
        }

      FloatVectorImageType::SizeType size = image->GetLargestPossibleRegion().GetSize();

      //Set the measurement vectors size if it's the first iteration
      if (imageId == 0)
        {
        mean.SetSize(nbBands);
        mean.Fill(0.);
        variance.SetSize(nbBands);
        variance.Fill(0.);
        }

      // Compute Statistics of each VectorImage
      StreamingStatisticsVImageFilterType::Pointer statsEstimator = StreamingStatisticsVImageFilterType::New();
      statsEstimator->SetInput(image);
      statsEstimator->Update();
      mean += statsEstimator->GetMean();
      for (unsigned int itBand = 0; itBand < nbBands; itBand++)
        {
        variance[itBand] += (size[0] * size[1] - 1) * (statsEstimator->GetCovariance())(itBand, itBand);
        }
      //Increment nbSamples
      nbSamples += size[0] * size[1] * nbBands;
      }

    //Divide by the number of input images to get the mean over all layers
    mean /= imageList->Size();
    //Apply the pooled variance formula
    variance /= (nbSamples - imageList->Size());

    MeasurementType stddev;
    stddev.SetSize(nbBands);
    stddev.Fill(0.);
    for (unsigned int i = 0; i < variance.GetSize(); ++i)
      {
      stddev[i] = vcl_sqrt(variance[i]);
      }

    if( HasValue( "out" )==true )
      {
      // Write the Statistics via the statistic writer
      typedef otb::StatisticsXMLFileWriter<MeasurementType> StatisticsWriter;
      StatisticsWriter::Pointer writer = StatisticsWriter::New();
      writer->SetFileName(GetParameterString("out"));
      writer->AddInput("mean", mean);
      writer->AddInput("stddev", stddev);
      writer->Update();
      }
    else
      {
      std::cout<<"Mean: "<<mean<<std::endl;
      std::cout<<"Standard Deviation: "<<stddev<<std::endl;
      }
  }

  itk::LightObject::Pointer m_FilterRef;
};

}
}

OTB_APPLICATION_EXPORT(otb::Wrapper::EstimateImagesStatistics)
<|endoftext|>


<commit_before>/*
* Copyright (c) 2016-2017 metaverse core developers (see MVS-AUTHORS).
* Copyright (C) 2013, 2016 Swirly Cloud Limited.
*
* This program is free software; you can redistribute it and/or modify it under the terms of the
* GNU General Public License as published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
* even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with this program; if
* not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/

#include <thread>
#include <sstream>
#include <metaverse/explorer/json_helper.hpp>
#include <metaverse/mgbubble/WsPushServ.hpp>
#include <metaverse/server/server_node.hpp>

namespace mgbubble {
    constexpr auto EV_VERSION     = "version";
    constexpr auto EV_SUBSCRIBE   = "subscribe";
    constexpr auto EV_UNSUBSCRIBE = "unsubscribe";
    constexpr auto EV_SUBSCRIBED  = "subscribed";
    constexpr auto EV_UNSUBSCRIBED = "unsubscribed";
    constexpr auto EV_PUBLISH     = "publish";
    constexpr auto EV_REQUEST     = "request";
    constexpr auto EV_RESPONSE    = "response";
    constexpr auto EV_MG_ERROR    = "error";
    constexpr auto EV_INFO        = "info";

    constexpr auto CH_BLOCK       = "block";
    constexpr auto CH_TRANSACTION = "tx";
}
namespace mgbubble {
using namespace bc;
using namespace libbitcoin;

void WsPushServ::run() {
    log::info(NAME) << "Websocket Service listen on " << node_.server_settings().websocket_listen;

    node_.subscribe_stop([this](const libbitcoin::code& ec) { stop(); });

    node_.subscribe_transaction_pool(
        std::bind(&WsPushServ::handle_transaction_pool,
            this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3));

    node_.subscribe_blockchain(
        std::bind(&WsPushServ::handle_blockchain_reorganization,
            this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, std::placeholders::_4));

    base::run();

    log::info(NAME) << "Websocket Service Stopped.";
}

bool WsPushServ::start()
{
    if (node_.server_settings().websocket_service_enabled == false)
        return true;
    if (!attach_notify())
        return false;
    return base::start();
}

void WsPushServ::spawn_to_mongoose(const std::function<void(uint64_t)>&& handler)
{
    auto msg = std::make_shared<WsEvent>(std::move(handler));
    struct mg_event ev { msg->hook() };
    if (!notify(ev))
        msg->unhook();
}

bool WsPushServ::handle_transaction_pool(const code& ec, const index_list&, message::transaction_message::ptr tx)
{
    if (stopped())
        return false;
    if (ec == (code)error::mock || ec == (code)error::service_stopped)
        return true;
    if (ec)
    {
        log::debug(NAME) << "Failure handling new transaction: " << ec.message();
        return true;
    }

    notify_transaction(0, null_hash, *tx);
    return true;
}

bool WsPushServ::handle_blockchain_reorganization(const code& ec, uint64_t fork_point, const block_list& new_blocks, const block_list&)
{
    if (stopped())
        return false;
    if (ec == (code)error::mock || ec == (code)error::service_stopped)
        return true;
    if (ec)
    {
        log::debug(NAME) << "Failure handling new block: " << ec.message();
        return true;
    }

    const auto fork_point32 = static_cast<uint32_t>(fork_point);

    notify_blocks(fork_point32, new_blocks);
    return true;
}

void WsPushServ::notify_blocks(uint32_t fork_point, const block_list& blocks)
{
    if (stopped())
        return;

    auto height = fork_point;

    for (const auto block : blocks)
        notify_block(height++, block);
}

void WsPushServ::notify_block(uint32_t height, const block::ptr block)
{
    if (stopped())
        return;

    const auto block_hash = block->header.hash();

    for (const auto& tx : block->transactions)
    {
        const auto tx_hash = tx.hash();

        notify_transaction(height, block_hash, tx);
    }
}

void WsPushServ::notify_transaction(uint32_t height, const hash_digest& block_hash, const transaction& tx)
{
    if (stopped() || tx.outputs.empty())
        return;

    std::map<std::weak_ptr<mg_connection>, std::vector<size_t>, std::owner_less<std::weak_ptr<mg_connection>>> subscribers;
    {
        std::lock_guard<std::mutex> guard(subscribers_lock_);
        if (subscribers_.size() == 0)
            return;
        for (auto& con : subscribers_)
        {
            if (!con.first.expired())
            {
                subscribers.insert(con);
            }
        }
        if (subscribers.size() != subscribers_.size())
            subscribers_ = subscribers;
    }

    /* ---------- may has subscribers ---------- */

    std::vector<size_t> tx_addrs;
    for (const auto& input : tx.inputs)
    {
        const auto address = payment_address::extract(input.script);
        if (address)
            tx_addrs.push_back(std::hash<payment_address>()(address));
    }

    for (const auto& output : tx.outputs)
    {
        const auto address = payment_address::extract(output.script);
        if (address)
            tx_addrs.push_back(std::hash<payment_address>()(address));
    }

    std::vector<std::weak_ptr<mg_connection>> notify_cons;
    for (auto& sub : subscribers)
    {
        auto& sub_addrs = sub.second;
        bool bnotify = sub_addrs.size() == 0 ? true : std::any_of(sub_addrs.begin(), sub_addrs.end(), [&tx_addrs](size_t addr_hash) {
            return tx_addrs.end() != std::find(tx_addrs.begin(), tx_addrs.end(), addr_hash);
        });

        if (bnotify)
            notify_cons.push_back(sub.first);
    }
    if (notify_cons.size() == 0)
        return;

    log::info(NAME) << " ******** notify_transaction: height [" << height << "]  ******** ";

    Json::Value root;
    root["event"] = EV_PUBLISH;
    root["channel"] = CH_TRANSACTION;
    root["result"] = explorer::config::json_helper().prop_list(tx, height, true);

    auto rep = std::make_shared<std::string>(std::move(root.toStyledString()));

    for (auto& con : notify_cons)
    {
        auto shared_con = con.lock();
        if (!shared_con)
            continue;

        spawn_to_mongoose([this, shared_con, rep](uint64_t id) {
            size_t active_connections = 0;
            auto* mgr = &this->mg_mgr();
            auto* notify_nc = shared_con.get();
            for (auto* nc = mg_next(mgr, NULL); nc != NULL; nc = mg_next(mgr, nc)) {
                if (!is_websocket(*nc) || is_listen_socket(*nc) || is_notify_socket(*nc))
                    continue;
                ++active_connections;
                if (notify_nc == nc)
                    send_frame(*nc, *rep);
            }
            if (active_connections != map_connections_.size())
                refresh_connections();
        });
    }
}

void WsPushServ::send_bad_response(struct mg_connection& nc, const char* message)
{
    Json::Value root;
    Json::Value result;
    result["code"] = 1000001;
    result["message"] = message ? message : "bad request";
    root["event"]  = EV_MG_ERROR;
    root["result"] = result;

    auto&& tmp = root.toStyledString();
    send_frame(nc, tmp.c_str(), tmp.size());
}

void WsPushServ::send_response(struct mg_connection& nc, const std::string& event, const std::string& channel)
{
    Json::Value root;
    root["event"] = event;
    root["channel"] = channel;

    auto&& tmp = root.toStyledString();
    send_frame(nc, tmp.c_str(), tmp.size());
}

void WsPushServ::refresh_connections()
{
    auto* mgr = &mg_mgr();
    std::unordered_map<void*, std::shared_ptr<mg_connection>> swap;
    for (auto* nc = mg_next(mgr, NULL); nc != NULL; nc = mg_next(mgr, nc)) {
        if (!is_websocket(*nc) || is_listen_socket(*nc) || is_notify_socket(*nc))
            continue;
        std::shared_ptr<struct mg_connection> con(nc, [](struct mg_connection* ptr) { (void)(ptr); });
        swap.emplace(&nc, con);
    }
    map_connections_.swap(swap);
}

void WsPushServ::on_ws_handshake_done_handler(struct mg_connection& nc)
{
    std::shared_ptr<struct mg_connection> con(&nc, [](struct mg_connection* ptr) { (void)(ptr); });
    map_connections_.emplace(&nc, con);

    std::string version("{\"event\": \"version\", " "\"result\": \"" MVS_VERSION "\"}");
    send_frame(nc, version);

    std::stringstream ss;
    Json::Value root;
    Json::Value connections;
    connections["connections"] = map_connections_.size();
    root["event"] = EV_INFO;
    root["result"] = connections;

    auto&& tmp = root.toStyledString();
    send_frame(nc, tmp);
}

void WsPushServ::on_ws_frame_handler(struct mg_connection& nc, websocket_message& msg)
{
    Json::Reader reader;
    Json::Value root;
    try {
        const char* begin = (const char*)msg.data;
        const char* end = begin + msg.size;
        if (!reader.parse(begin, end, root) || !root.isObject()
            || !root["event"].isString() || !root["channel"].isString() || !root["address"].isString()) {
            stringstream ss;
            ss << "parse request error, "
                << reader.getFormattedErrorMessages();
            throw std::runtime_error(ss.str());
            return;
        }

        auto event = root["event"].asString();
        auto channel = root["channel"].asString();
        if ((event == EV_SUBSCRIBE) && (channel == CH_TRANSACTION)) {
            auto short_addr = root["address"].asString();
            auto pay_addr = payment_address(short_addr);
            if (!short_addr.empty() && !pay_addr) {
                send_bad_response(nc, "invalid address.");
            }
            else {
                size_t hash_addr = short_addr.empty() ? 0 : std::hash<payment_address>()(pay_addr);
                auto it = map_connections_.find(&nc);
                if (it != map_connections_.end()) {
                    std::lock_guard<std::mutex> guard(subscribers_lock_);
                    std::weak_ptr<struct mg_connection> week_con(it->second);
                    auto sub_it = subscribers_.find(week_con);
                    if (sub_it != subscribers_.end()) {
                        auto& sub_list = sub_it->second;
                        if (hash_addr == 0) {
                            sub_list.clear();
                            send_response(nc, EV_SUBSCRIBED, channel);
                        }
                        else {
                            if (sub_list.end() == std::find(sub_list.begin(), sub_list.end(), hash_addr)) {
                                sub_list.push_back(hash_addr);
                                send_response(nc, EV_SUBSCRIBED, channel);
                            }
                            else {
                                send_bad_response(nc, "address already subscribed.");
                            }
                        }
                    }
                    else {
                        if (hash_addr == 0)
                            subscribers_.insert({ week_con, {} });
                        else
                            subscribers_.insert({ week_con, { hash_addr } });
                        send_response(nc, EV_SUBSCRIBED, channel);
                    }
                }
                else {
                    send_bad_response(nc, "connection lost.");
                }
            }
        }
        else if ((event == EV_UNSUBSCRIBE) && (channel == CH_TRANSACTION)) {
            auto it = map_connections_.find(&nc);
            if (it != map_connections_.end()) {
                std::lock_guard<std::mutex> guard(subscribers_lock_);
                std::weak_ptr<struct mg_connection> week_con(it->second);
                subscribers_.erase(week_con);
                send_response(nc, EV_UNSUBSCRIBED, channel);
            }
            else {
                send_bad_response(nc, "no subscription.");
            }
        }
        else {
            send_bad_response(nc, "request not support.");
        }
    }
    catch (std::exception& e) {
        log::info(NAME) << "on on_ws_frame_handler: " << e.what();
        send_bad_response(nc);
    }
}

void WsPushServ::on_close_handler(struct mg_connection& nc)
{
    if (is_websocket(nc))
    {
        map_connections_.erase(&nc);
    }
}

void WsPushServ::on_broadcast(struct mg_connection& nc, const char* ev_data)
{
    if (is_listen_socket(nc) || is_notify_socket(nc))
        return;

    send_frame(nc, ev_data, strlen(ev_data));
}

void WsPushServ::on_send_handler(struct mg_connection& nc, int bytes_transfered)
{
}

void WsPushServ::on_notify_handler(struct mg_connection& nc, struct mg_event& ev)
{
    static uint64_t api_call_counter = 0;

    if (ev.data == nullptr)
        return;

    auto& msg = *(WsEvent*)ev.data;
    msg(++api_call_counter);
}

}
<commit_msg>compiling issue, compatible for osx<commit_after>/*
* Copyright (c) 2016-2017 metaverse core developers (see MVS-AUTHORS).
* Copyright (C) 2013, 2016 Swirly Cloud Limited.
*
* This program is free software; you can redistribute it and/or modify it under the terms of the
* GNU General Public License as published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
* even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with this program; if
* not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/

#include <thread>
#include <sstream>
#include <metaverse/explorer/json_helper.hpp>
#include <metaverse/mgbubble/WsPushServ.hpp>
#include <metaverse/server/server_node.hpp>

namespace mgbubble {
    constexpr auto EV_VERSION     = "version";
    constexpr auto EV_SUBSCRIBE   = "subscribe";
    constexpr auto EV_UNSUBSCRIBE = "unsubscribe";
    constexpr auto EV_SUBSCRIBED  = "subscribed";
    constexpr auto EV_UNSUBSCRIBED = "unsubscribed";
    constexpr auto EV_PUBLISH     = "publish";
    constexpr auto EV_REQUEST     = "request";
    constexpr auto EV_RESPONSE    = "response";
    constexpr auto EV_MG_ERROR    = "error";
    constexpr auto EV_INFO        = "info";

    constexpr auto CH_BLOCK       = "block";
    constexpr auto CH_TRANSACTION = "tx";
}
namespace mgbubble {
using namespace bc;
using namespace libbitcoin;

void WsPushServ::run() {
    log::info(NAME) << "Websocket Service listen on " << node_.server_settings().websocket_listen;

    node_.subscribe_stop([this](const libbitcoin::code& ec) { stop(); });

    node_.subscribe_transaction_pool(
        std::bind(&WsPushServ::handle_transaction_pool,
            this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3));

    node_.subscribe_blockchain(
        std::bind(&WsPushServ::handle_blockchain_reorganization,
            this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, std::placeholders::_4));

    base::run();

    log::info(NAME) << "Websocket Service Stopped.";
}

bool WsPushServ::start()
{
    if (node_.server_settings().websocket_service_enabled == false)
        return true;
    if (!attach_notify())
        return false;
    return base::start();
}

void WsPushServ::spawn_to_mongoose(const std::function<void(uint64_t)>&& handler)
{
    auto msg = std::make_shared<WsEvent>(std::move(handler));
    struct mg_event ev { msg->hook() };
    if (!notify(ev))
        msg->unhook();
}

bool WsPushServ::handle_transaction_pool(const code& ec, const index_list&, message::transaction_message::ptr tx)
{
    if (stopped())
        return false;
    if (ec == (code)error::mock || ec == (code)error::service_stopped)
        return true;
    if (ec)
    {
        log::debug(NAME) << "Failure handling new transaction: " << ec.message();
        return true;
    }

    notify_transaction(0, null_hash, *tx);
    return true;
}

bool WsPushServ::handle_blockchain_reorganization(const code& ec, uint64_t fork_point, const block_list& new_blocks, const block_list&)
{
    if (stopped())
        return false;
    if (ec == (code)error::mock || ec == (code)error::service_stopped)
        return true;
    if (ec)
    {
        log::debug(NAME) << "Failure handling new block: " << ec.message();
        return true;
    }

    const auto fork_point32 = static_cast<uint32_t>(fork_point);

    notify_blocks(fork_point32, new_blocks);
    return true;
}

void WsPushServ::notify_blocks(uint32_t fork_point, const block_list& blocks)
{
    if (stopped())
        return;

    auto height = fork_point;

    for (const auto block : blocks)
        notify_block(height++, block);
}

void WsPushServ::notify_block(uint32_t height, const block::ptr block)
{
    if (stopped())
        return;

    const auto block_hash = block->header.hash();

    for (const auto& tx : block->transactions)
    {
        const auto tx_hash = tx.hash();

        notify_transaction(height, block_hash, tx);
    }
}

void WsPushServ::notify_transaction(uint32_t height, const hash_digest& block_hash, const transaction& tx)
{
    if (stopped() || tx.outputs.empty())
        return;

    std::map<std::weak_ptr<mg_connection>, std::vector<size_t>, std::owner_less<std::weak_ptr<mg_connection>>> subscribers;
    {
        std::lock_guard<std::mutex> guard(subscribers_lock_);
        if (subscribers_.size() == 0)
            return;
        for (auto& con : subscribers_)
        {
            if (!con.first.expired())
            {
                subscribers.insert(con);
            }
        }
        if (subscribers.size() != subscribers_.size())
            subscribers_ = subscribers;
    }

    /* ---------- may has subscribers ---------- */

    std::vector<size_t> tx_addrs;
    for (const auto& input : tx.inputs)
    {
        const auto address = payment_address::extract(input.script);
        if (address)
            tx_addrs.push_back(std::hash<payment_address>()(address));
    }

    for (const auto& output : tx.outputs)
    {
        const auto address = payment_address::extract(output.script);
        if (address)
            tx_addrs.push_back(std::hash<payment_address>()(address));
    }

    std::vector<std::weak_ptr<mg_connection>> notify_cons;
    for (auto& sub : subscribers)
    {
        auto& sub_addrs = sub.second;
        bool bnotify = sub_addrs.size() == 0 ? true : std::any_of(sub_addrs.begin(), sub_addrs.end(), [&tx_addrs](size_t addr_hash) {
            return tx_addrs.end() != std::find(tx_addrs.begin(), tx_addrs.end(), addr_hash);
        });

        if (bnotify)
            notify_cons.push_back(sub.first);
    }
    if (notify_cons.size() == 0)
        return;

    log::info(NAME) << " ******** notify_transaction: height [" << height << "]  ******** ";

    Json::Value root;
    root["event"] = EV_PUBLISH;
    root["channel"] = CH_TRANSACTION;
    root["result"] = explorer::config::json_helper().prop_list(tx, height, true);

    auto rep = std::make_shared<std::string>(root.toStyledString());

    for (auto& con : notify_cons)
    {
        auto shared_con = con.lock();
        if (!shared_con)
            continue;

        spawn_to_mongoose([this, shared_con, rep](uint64_t id) {
            size_t active_connections = 0;
            auto* mgr = &this->mg_mgr();
            auto* notify_nc = shared_con.get();
            for (auto* nc = mg_next(mgr, NULL); nc != NULL; nc = mg_next(mgr, nc)) {
                if (!is_websocket(*nc) || is_listen_socket(*nc) || is_notify_socket(*nc))
                    continue;
                ++active_connections;
                if (notify_nc == nc)
                    send_frame(*nc, *rep);
            }
            if (active_connections != map_connections_.size())
                refresh_connections();
        });
    }
}

void WsPushServ::send_bad_response(struct mg_connection& nc, const char* message)
{
    Json::Value root;
    Json::Value result;
    result["code"] = 1000001;
    result["message"] = message ? message : "bad request";
    root["event"]  = EV_MG_ERROR;
    root["result"] = result;

    auto&& tmp = root.toStyledString();
    send_frame(nc, tmp.c_str(), tmp.size());
}

void WsPushServ::send_response(struct mg_connection& nc, const std::string& event, const std::string& channel)
{
    Json::Value root;
    root["event"] = event;
    root["channel"] = channel;

    auto&& tmp = root.toStyledString();
    send_frame(nc, tmp.c_str(), tmp.size());
}

void WsPushServ::refresh_connections()
{
    auto* mgr = &mg_mgr();
    std::unordered_map<void*, std::shared_ptr<mg_connection>> swap;
    for (auto* nc = mg_next(mgr, NULL); nc != NULL; nc = mg_next(mgr, nc)) {
        if (!is_websocket(*nc) || is_listen_socket(*nc) || is_notify_socket(*nc))
            continue;
        std::shared_ptr<struct mg_connection> con(nc, [](struct mg_connection* ptr) { (void)(ptr); });
        swap.emplace(&nc, con);
    }
    map_connections_.swap(swap);
}

void WsPushServ::on_ws_handshake_done_handler(struct mg_connection& nc)
{
    std::shared_ptr<struct mg_connection> con(&nc, [](struct mg_connection* ptr) { (void)(ptr); });
    map_connections_.emplace(&nc, con);

    std::string version("{\"event\": \"version\", " "\"result\": \"" MVS_VERSION "\"}");
    send_frame(nc, version);

    std::stringstream ss;
    Json::Value root;
    Json::Value connections;
    connections["connections"] = static_cast<uint64_t>(map_connections_.size());
    root["event"] = EV_INFO;
    root["result"] = connections;

    auto&& tmp = root.toStyledString();
    send_frame(nc, tmp);
}

void WsPushServ::on_ws_frame_handler(struct mg_connection& nc, websocket_message& msg)
{
    Json::Reader reader;
    Json::Value root;
    try {
        const char* begin = (const char*)msg.data;
        const char* end = begin + msg.size;
        if (!reader.parse(begin, end, root) || !root.isObject()
            || !root["event"].isString() || !root["channel"].isString() || !root["address"].isString()) {
            stringstream ss;
            ss << "parse request error, "
                << reader.getFormattedErrorMessages();
            throw std::runtime_error(ss.str());
            return;
        }

        auto event = root["event"].asString();
        auto channel = root["channel"].asString();
        if ((event == EV_SUBSCRIBE) && (channel == CH_TRANSACTION)) {
            auto short_addr = root["address"].asString();
            auto pay_addr = payment_address(short_addr);
            if (!short_addr.empty() && !pay_addr) {
                send_bad_response(nc, "invalid address.");
            }
            else {
                size_t hash_addr = short_addr.empty() ? 0 : std::hash<payment_address>()(pay_addr);
                auto it = map_connections_.find(&nc);
                if (it != map_connections_.end()) {
                    std::lock_guard<std::mutex> guard(subscribers_lock_);
                    std::weak_ptr<struct mg_connection> week_con(it->second);
                    auto sub_it = subscribers_.find(week_con);
                    if (sub_it != subscribers_.end()) {
                        auto& sub_list = sub_it->second;
                        if (hash_addr == 0) {
                            sub_list.clear();
                            send_response(nc, EV_SUBSCRIBED, channel);
                        }
                        else {
                            if (sub_list.end() == std::find(sub_list.begin(), sub_list.end(), hash_addr)) {
                                sub_list.push_back(hash_addr);
                                send_response(nc, EV_SUBSCRIBED, channel);
                            }
                            else {
                                send_bad_response(nc, "address already subscribed.");
                            }
                        }
                    }
                    else {
                        if (hash_addr == 0)
                            subscribers_.insert({ week_con, {} });
                        else
                            subscribers_.insert({ week_con, { hash_addr } });
                        send_response(nc, EV_SUBSCRIBED, channel);
                    }
                }
                else {
                    send_bad_response(nc, "connection lost.");
                }
            }
        }
        else if ((event == EV_UNSUBSCRIBE) && (channel == CH_TRANSACTION)) {
            auto it = map_connections_.find(&nc);
            if (it != map_connections_.end()) {
                std::lock_guard<std::mutex> guard(subscribers_lock_);
                std::weak_ptr<struct mg_connection> week_con(it->second);
                subscribers_.erase(week_con);
                send_response(nc, EV_UNSUBSCRIBED, channel);
            }
            else {
                send_bad_response(nc, "no subscription.");
            }
        }
        else {
            send_bad_response(nc, "request not support.");
        }
    }
    catch (std::exception& e) {
        log::info(NAME) << "on on_ws_frame_handler: " << e.what();
        send_bad_response(nc);
    }
}

void WsPushServ::on_close_handler(struct mg_connection& nc)
{
    if (is_websocket(nc))
    {
        map_connections_.erase(&nc);
    }
}

void WsPushServ::on_broadcast(struct mg_connection& nc, const char* ev_data)
{
    if (is_listen_socket(nc) || is_notify_socket(nc))
        return;

    send_frame(nc, ev_data, strlen(ev_data));
}

void WsPushServ::on_send_handler(struct mg_connection& nc, int bytes_transfered)
{
}

void WsPushServ::on_notify_handler(struct mg_connection& nc, struct mg_event& ev)
{
    static uint64_t api_call_counter = 0;

    if (ev.data == nullptr)
        return;

    auto& msg = *(WsEvent*)ev.data;
    msg(++api_call_counter);
}

}
<|endoftext|>


<commit_before>#include "upsample.h"

void upsample(const std::vector<complex> &input, float n, std::vector<complex> &output) {
    int M = input.size();
    int N;
    for (int i=0; i<24; i++) {
        if (M <= (1<<i)) {
            N = 1<<i;
            break;
        }
    }
    int L = N + int((n-1)*N);
    std::vector<complex> x(input);
    x.resize(N);
    fft(&x[0], N);
    x.resize(L);
    for (int i=N-1; i>=N/2; i--)
        x[L-N+i] = x[i];
    for (int i=N/2; i<L-N/2; i++)
        x[i] = 0;
    ifft(&x[0], N);
    output.assign(x.begin(), x.begin() + int(M*n));
}
<commit_msg>Fixed upsample<commit_after>#include "upsample.h"

void upsample(const std::vector<complex> &input, float n, std::vector<complex> &output) {
    int M = input.size();
    int N;
    for (int i=0; i<24; i++) {
        if (M <= (1<<i)) {
            N = 1<<i;
            break;
        }
    }
    int L = N + int((n-1)*N);
    std::vector<complex> x(input);
    x.resize(N);
    fft(&x[0], N);
    x.resize(L);
    for (int i=N-1; i>=N/2; i--)
        x[L-N+i] = x[i];
    for (int i=N/2; i<L-N/2; i++)
        x[i] = 0;
    ifft(&x[0], L);
    output.assign(x.begin(), x.begin() + int(M*n));
}
<|endoftext|>