<commit_before>#include "place.hpp"

#include <fstream>
#include <string>
#include <memory>
#include <functional>
#include <limits>

#ifdef __OMP
#include <omp.h>
#endif

#include "file_io.hpp"
#include "jplace_util.hpp"
#include "stringify.hpp"
#include "set_manipulators.hpp"
#include "logging.hpp"
#include "Tiny_Tree.hpp"
#include "mpihead.hpp"
#include "pll_util.hpp"
#include "epa_pll_util.hpp"
#include "Timer.hpp"
#include "Work.hpp"
#include "schedule.hpp"
#include "Lookup_Store.hpp"
#include "Pipeline.hpp"
#include "MSA.hpp"
#include "Work.hpp"
#include "Sample.hpp"
#include "Binary_Fasta.hpp"

#ifdef __MPI
#include "epa_mpi_util.hpp"
#endif

template <class T>
static void place(const Work& to_place,
                  MSA& msa,
                  Tree& reference_tree,
                  const std::vector<pll_unode_t *>& branches,
                  Sample<T>& sample,
                  bool do_blo,
                  const Options& options,
                  std::shared_ptr<Lookup_Store>& lookup_store,
                  const size_t seq_id_offset=0)
{

#ifdef __OMP
  const unsigned int num_threads = options.num_threads ? options.num_threads : omp_get_max_threads();
  omp_set_num_threads(num_threads);
  LOG_DBG << "Using threads: " << num_threads;
  LOG_DBG << "Max threads: " << omp_get_max_threads();
#else
  const unsigned int num_threads = 1;
#endif

  // split the sample structure such that the parts are thread-local
  std::vector<Sample<T>> sample_parts(num_threads);
  std::vector<Work> work_parts;
  split(to_place, work_parts, num_threads);

  // work seperately
#ifdef __OMP
  #pragma omp parallel for schedule(dynamic)
#endif
  for (size_t i = 0; i < work_parts.size(); ++i) {
    auto prev_branch_id = std::numeric_limits<size_t>::max();

    std::shared_ptr<Tiny_Tree> branch(nullptr);

    for (auto it : work_parts[i]) {
      const auto branch_id = it.branch_id;
      const auto seq_id = it.sequence_id;

      if ((branch_id != prev_branch_id) or not branch) {
        branch = std::make_shared<Tiny_Tree>(branches[branch_id],
                                             branch_id,
                                             reference_tree,
                                             do_blo,
                                             options,
                                             lookup_store);
      }

      sample_parts[i].add_placement(seq_id_offset + seq_id,
                                    msa[seq_id].header(),
                                    branch->place(msa[seq_id]));

      prev_branch_id = branch_id;
    }
  }
  // merge samples back
  merge(sample, sample_parts);
}

void pipeline_place(Tree& reference_tree,
                    const std::string& query_file,
                    const std::string& outdir,
                    const Options& options,
                    const std::string& invocation)
{
  LOG_INFO << "WARNING! THIS FUNCTION IS EXPERIMENTAL!" << std::endl;

  // Timer<> flight_time;
  std::ofstream flight_file(outdir + "stat");

  std::string status_file_name(outdir + "pepa.status");
  std::ofstream trunc_status_file(status_file_name, std::ofstream::trunc);

  std::vector<std::string> part_names;

  const auto chunk_size = options.chunk_size;
  LOG_DBG << "Chunk size: " << chunk_size << std::endl;

  const auto num_branches = reference_tree.nums().branches;

  // get all edges
  std::vector<pll_unode_t *> branches(num_branches);
  auto num_traversed_branches = utree_query_branches(reference_tree.tree(), &branches[0]);
  if (num_traversed_branches != num_branches) {
    throw std::runtime_error{"Traversing the utree went wrong during pipeline startup!"};
  }

  unsigned int chunk_num = 0;

  auto lookups =
    std::make_shared<Lookup_Store>(num_branches, reference_tree.partition()->states);

  Work all_work(std::make_pair(0, num_branches), std::make_pair(0, chunk_size));

  MSA chunk;
  Binary_Fasta_Reader reader(query_file);

  size_t num_sequences = 0;

  // create output file
  std::ofstream outfile;

  using Slim_Sample = Sample<Slim_Placement>;
  using Sample      = Sample<Placement>;

  // ============ LAMBDAS ============================

  // only on one rank, only once at the beginning of pipeline
  auto init_pipe_func = [&]() -> void {
    outfile.open(outdir + "epa_result.jplace");
    auto newick_string = get_numbered_newick_string(reference_tree.tree());
    outfile << init_jplace_string(newick_string);
  };

  auto perloop_prehook = [&]() -> void {
    LOG_DBG << "INGESTING - READING" << std::endl;
    num_sequences = reader.read_next(chunk, chunk_size);
    ++chunk_num;
  };

  auto ingestion = [&](VoidToken&) -> Work {
    LOG_DBG << "INGESTING - CREATING WORK" << std::endl;
    if (num_sequences <= 0) {
      Work work;
      work.is_last(true);
      return work;
    } else if (num_sequences < chunk_size) {
      return Work(std::make_pair(0, num_branches), std::make_pair(0, num_sequences));
    } else {
      return all_work;
    }
  };

  auto preplacement = [&](Work& work) -> Slim_Sample {
    LOG_DBG << "PREPLACING" << std::endl;

    Slim_Sample result;

    place(work,
          chunk,
          reference_tree,
          branches,
          result,
          false,
          options,
          lookups);

    return result;
  };

  // auto ingest_preplace = [&](VoidToken&) -> Slim_Sample {
  //   LOG_DBG << "INGEST & PREPLACE" << std::endl;

  //   Slim_Sample result;

  //   if (num_sequences <= 0) {
  //     result.is_last(true);
  //     return result;
  //   } else if (num_sequences < chunk_size) {
  //     Work work(std::make_pair(0, num_branches), std::make_pair(0, num_sequences));
  //     std::vector<Work> parts
  //   } else {
  //     return all_work;
  //   }

  //   place(work,
  //         chunk,
  //         reference_tree,
  //         branches,
  //         result,
  //         false,
  //         options,
  //         lookups);

  //   return result;
  // };

  auto candidate_selection = [&](Slim_Sample& slim) -> Work {
    LOG_DBG << "SELECTING CANDIDATES" << std::endl;

    Sample sample(slim);

    compute_and_set_lwr(sample);

    if (options.prescoring_by_percentage) {
      discard_bottom_x_percent(sample,
                              (1.0 - options.prescoring_threshold));
    } else {
      discard_by_accumulated_threshold(sample,
                                      options.prescoring_threshold,
                                      options.filter_min,
                                      options.filter_max);
    }

    return Work(sample);
  };

  auto thorough_placement = [&](Work& work) -> Sample {
    LOG_DBG << "BLO PLACEMENT" << std::endl;

    Sample result;
    place(work,
          chunk,
          reference_tree,
          branches,
          result,
          true,
          options,
          lookups
    );
    return result;
  };

  auto write_result = [&](Sample& sample) -> VoidToken {
    LOG_DBG << "WRITING" << std::endl;

    compute_and_set_lwr(sample);
    if (options.acc_threshold) {
      LOG_DBG << "Filtering by accumulated threshold: " << options.support_threshold << std::endl;
      discard_by_accumulated_threshold( sample,
                                        options.support_threshold,
                                        options.filter_min,
                                        options.filter_max);
    } else {
      LOG_DBG << "Filtering placements below threshold: " << options.support_threshold << std::endl;
      discard_by_support_threshold( sample,
                                    options.support_threshold,
                                    options.filter_min,
                                    options.filter_max);
    }

    // write results of current last stage aggregator node to a part file
    if (sample.size()) {
      if (chunk_num > 1) {
        outfile << ",";
      }
      outfile << sample_to_jplace_string(sample);
      // std::string part_file_name(outdir + "epa." + std::to_string(local_rank)
      //   + "." + std::to_string(chunk_num) + ".part");
      // std::ofstream part_file(part_file_name);
      // part_file << sample_to_jplace_string(sample, chunk);
      // part_names.push_back(part_file_name);
      // part_file.close();

      // // TODO for MPI, somehow ensure this code is only run on the stage foreman

      // std::ofstream status_file(status_file_name, std::ofstream::app);
      // status_file << chunk_num << ":" << chunk_size << " [";
      // for (size_t i = 0; i < part_names.size(); ++i) {
      //   status_file << part_names[i];
      //   if (i < part_names.size() - 1) status_file << ",";
      // }
      // status_file << "]" << std::endl;
      // part_names.clear();
    }

    LOG_INFO << chunk_num * chunk_size  << " Sequences done!";

    return VoidToken();
  };

  // only on one rank, only once at the end of the pipeline
  auto finalize_pipe_func = [&]() -> void {
    LOG_INFO << "Output file: " << outdir + "epa_result.jplace";
    outfile << finalize_jplace_string(invocation);
    outfile.close();
  };

  if (options.prescoring) {
    auto pipe = make_pipeline(ingestion, perloop_prehook, init_pipe_func, finalize_pipe_func)
      .push(preplacement)
      .push(candidate_selection)
      .push(thorough_placement)
      .push(write_result);

    pipe.process();
  } else {
    auto pipe = make_pipeline(ingestion, perloop_prehook, init_pipe_func, finalize_pipe_func)
      .push(thorough_placement)
      .push(write_result);

    pipe.process();
  }
}

void simple_mpi(Tree& reference_tree,
                const std::string& query_file,
                const std::string& outdir,
                const Options& options,
                const std::string& invocation)
{
  LOG_INFO << "WARNING! THIS FUNCTION IS EXPERIMENTAL!" << std::endl;

  // Timer<> flight_time;
  std::ofstream flight_file(outdir + "stat");

  std::string status_file_name(outdir + "pepa.status");
  std::ofstream trunc_status_file(status_file_name, std::ofstream::trunc);

  std::vector<std::string> part_names;

  const auto num_branches = reference_tree.nums().branches;

  // get all edges
  std::vector<pll_unode_t *> branches(num_branches);
  auto num_traversed_branches = utree_query_branches(reference_tree.tree(), &branches[0]);
  if (num_traversed_branches != num_branches) {
    throw std::runtime_error{"Traversing the utree went wrong during pipeline startup!"};
  }

  auto lookups =
    std::make_shared<Lookup_Store>(num_branches, reference_tree.partition()->states);

  // some MPI prep
  int local_rank = 0;
  int num_ranks = 1;

  MPI_COMM_RANK(MPI_COMM_WORLD, &local_rank);
  MPI_COMM_SIZE(MPI_COMM_WORLD, &num_ranks);

  LOG_DBG << "Number of ranks: " << num_ranks;

  std::vector<int> all_ranks(num_ranks);
  for (int i = 0; i < num_ranks; ++i) {
    all_ranks[i] = i;
  }

  Binary_Fasta_Reader reader(query_file);

  // how many should each rank read?
  const size_t part_size = ceil(reader.num_sequences() / static_cast<double>(num_ranks));
  LOG_DBG << "Number of sequences per rank: " << part_size;

  // read only the locally relevant part of the queries
  // ... by skipping the appropriate amount
  const size_t local_rank_seq_offset = part_size * local_rank;
  reader.skip_to_sequence( local_rank_seq_offset );
  // and limiting the reading to the given window
  reader.constrain(part_size);

  size_t num_sequences = options.chunk_size;
  Work all_work(std::make_pair(0, num_branches), std::make_pair(0, num_sequences));

  size_t chunk_num = 1;

  using Sample = Sample<Placement>;
  Sample result;
  MSA chunk;
  size_t sequences_done = 0; // not just for info output!
  while ( (num_sequences = reader.read_next(chunk, options.chunk_size) ) ) {

    assert(chunk.size() == num_sequences);

    LOG_DBG << "num_sequences: " << num_sequences << std::endl;

    const size_t seq_id_offset = sequences_done + local_rank_seq_offset;

    if (num_sequences < options.chunk_size) {
      all_work = Work(std::make_pair(0, num_branches), std::make_pair(0, num_sequences));
    }

    Sample preplace;

    LOG_DBG << "Preplacement." << std::endl;
    place(all_work,
          chunk,
          reference_tree,
          branches,
          preplace,
          false,
          options,
          lookups);

    // Candidate Selection
    LOG_DBG << "Selecting candidates." << std::endl;
    compute_and_set_lwr(preplace);

    if (options.prescoring_by_percentage) {
      discard_bottom_x_percent(preplace,
                              (1.0 - options.prescoring_threshold));
    } else {
      discard_by_accumulated_threshold( preplace,
                                        options.prescoring_threshold,
                                        options.filter_min,
                                        options.filter_max);
    }

    Work blo_work(preplace);

    Sample blo_sample;

    // BLO placement
    LOG_DBG << "BLO Placement." << std::endl;
    place(blo_work,
          chunk,
          reference_tree,
          branches,
          blo_sample,
          true,
          options,
          lookups,
          seq_id_offset);

    // Output
    compute_and_set_lwr(blo_sample);
    if (options.acc_threshold) {
      LOG_DBG << "Filtering by accumulated threshold: " << options.support_threshold << std::endl;
      discard_by_accumulated_threshold( blo_sample,
                                        options.support_threshold,
                                        options.filter_min,
                                        options.filter_max);
    } else {
      LOG_DBG << "Filtering placements below threshold: " << options.support_threshold << std::endl;
      discard_by_support_threshold( blo_sample,
                                    options.support_threshold,
                                    options.filter_min,
                                    options.filter_max);
    }

    merge(result, blo_sample);

    sequences_done += num_sequences;
    LOG_INFO << sequences_done  << " Sequences done!";
    ++chunk_num;
  }

#ifdef __MPI
  // send to output: on rank <designated_writer>
  LOG_DBG << "Gathering results on Rank " << 0;
  Timer<> dummy;
  epa_mpi_gather(result, 0, all_ranks, local_rank, dummy);
#endif //__MPI

  if (local_rank == 0) {
    // create output file
    LOG_INFO << "Output file: " << outdir + "epa_result.jplace";
    std::ofstream outfile;
    outfile.open(outdir + "epa_result.jplace");
    outfile << init_jplace_string(
      get_numbered_newick_string(reference_tree.tree()));
    outfile << sample_to_jplace_string(result);
    outfile << finalize_jplace_string(invocation);
    outfile.close();
  }

  MPI_BARRIER(MPI_COMM_WORLD);
}

<commit_msg>switched per-chunk result merging over to the faster move-merge<commit_after>#include "place.hpp"

#include <fstream>
#include <string>
#include <memory>
#include <functional>
#include <limits>

#ifdef __OMP
#include <omp.h>
#endif

#include "file_io.hpp"
#include "jplace_util.hpp"
#include "stringify.hpp"
#include "set_manipulators.hpp"
#include "logging.hpp"
#include "Tiny_Tree.hpp"
#include "mpihead.hpp"
#include "pll_util.hpp"
#include "epa_pll_util.hpp"
#include "Timer.hpp"
#include "Work.hpp"
#include "schedule.hpp"
#include "Lookup_Store.hpp"
#include "Pipeline.hpp"
#include "MSA.hpp"
#include "Work.hpp"
#include "Sample.hpp"
#include "Binary_Fasta.hpp"

#ifdef __MPI
#include "epa_mpi_util.hpp"
#endif

template <class T>
static void place(const Work& to_place,
                  MSA& msa,
                  Tree& reference_tree,
                  const std::vector<pll_unode_t *>& branches,
                  Sample<T>& sample,
                  bool do_blo,
                  const Options& options,
                  std::shared_ptr<Lookup_Store>& lookup_store,
                  const size_t seq_id_offset=0)
{

#ifdef __OMP
  const unsigned int num_threads = options.num_threads ? options.num_threads : omp_get_max_threads();
  omp_set_num_threads(num_threads);
  LOG_DBG << "Using threads: " << num_threads;
  LOG_DBG << "Max threads: " << omp_get_max_threads();
#else
  const unsigned int num_threads = 1;
#endif

  // split the sample structure such that the parts are thread-local
  std::vector<Sample<T>> sample_parts(num_threads);
  std::vector<Work> work_parts;
  split(to_place, work_parts, num_threads);

  // work seperately
#ifdef __OMP
  #pragma omp parallel for schedule(dynamic)
#endif
  for (size_t i = 0; i < work_parts.size(); ++i) {
    auto prev_branch_id = std::numeric_limits<size_t>::max();

    std::shared_ptr<Tiny_Tree> branch(nullptr);

    for (const auto& it : work_parts[i]) {
      const auto branch_id = it.branch_id;
      const auto seq_id = it.sequence_id;

      if ((branch_id != prev_branch_id) or not branch) {
        branch = std::make_shared<Tiny_Tree>(branches[branch_id],
                                             branch_id,
                                             reference_tree,
                                             do_blo,
                                             options,
                                             lookup_store);
      }

      sample_parts[i].add_placement(seq_id_offset + seq_id,
                                    msa[seq_id].header(),
                                    branch->place(msa[seq_id]));

      prev_branch_id = branch_id;
    }
  }
  // merge samples back
  merge(sample, sample_parts);
}

void pipeline_place(Tree& reference_tree,
                    const std::string& query_file,
                    const std::string& outdir,
                    const Options& options,
                    const std::string& invocation)
{
  LOG_INFO << "WARNING! THIS FUNCTION IS EXPERIMENTAL!" << std::endl;

  // Timer<> flight_time;
  std::ofstream flight_file(outdir + "stat");

  std::string status_file_name(outdir + "pepa.status");
  std::ofstream trunc_status_file(status_file_name, std::ofstream::trunc);

  std::vector<std::string> part_names;

  const auto chunk_size = options.chunk_size;
  LOG_DBG << "Chunk size: " << chunk_size << std::endl;

  const auto num_branches = reference_tree.nums().branches;

  // get all edges
  std::vector<pll_unode_t *> branches(num_branches);
  auto num_traversed_branches = utree_query_branches(reference_tree.tree(), &branches[0]);
  if (num_traversed_branches != num_branches) {
    throw std::runtime_error{"Traversing the utree went wrong during pipeline startup!"};
  }

  unsigned int chunk_num = 0;

  auto lookups =
    std::make_shared<Lookup_Store>(num_branches, reference_tree.partition()->states);

  Work all_work(std::make_pair(0, num_branches), std::make_pair(0, chunk_size));

  MSA chunk;
  Binary_Fasta_Reader reader(query_file);

  size_t num_sequences = 0;

  // create output file
  std::ofstream outfile;

  using Slim_Sample = Sample<Slim_Placement>;
  using Sample      = Sample<Placement>;

  // ============ LAMBDAS ============================

  // only on one rank, only once at the beginning of pipeline
  auto init_pipe_func = [&]() -> void {
    outfile.open(outdir + "epa_result.jplace");
    auto newick_string = get_numbered_newick_string(reference_tree.tree());
    outfile << init_jplace_string(newick_string);
  };

  auto perloop_prehook = [&]() -> void {
    LOG_DBG << "INGESTING - READING" << std::endl;
    num_sequences = reader.read_next(chunk, chunk_size);
    ++chunk_num;
  };

  auto ingestion = [&](VoidToken&) -> Work {
    LOG_DBG << "INGESTING - CREATING WORK" << std::endl;
    if (num_sequences <= 0) {
      Work work;
      work.is_last(true);
      return work;
    } else if (num_sequences < chunk_size) {
      return Work(std::make_pair(0, num_branches), std::make_pair(0, num_sequences));
    } else {
      return all_work;
    }
  };

  auto preplacement = [&](Work& work) -> Slim_Sample {
    LOG_DBG << "PREPLACING" << std::endl;

    Slim_Sample result;

    place(work,
          chunk,
          reference_tree,
          branches,
          result,
          false,
          options,
          lookups);

    return result;
  };

  // auto ingest_preplace = [&](VoidToken&) -> Slim_Sample {
  //   LOG_DBG << "INGEST & PREPLACE" << std::endl;

  //   Slim_Sample result;

  //   if (num_sequences <= 0) {
  //     result.is_last(true);
  //     return result;
  //   } else if (num_sequences < chunk_size) {
  //     Work work(std::make_pair(0, num_branches), std::make_pair(0, num_sequences));
  //     std::vector<Work> parts
  //   } else {
  //     return all_work;
  //   }

  //   place(work,
  //         chunk,
  //         reference_tree,
  //         branches,
  //         result,
  //         false,
  //         options,
  //         lookups);

  //   return result;
  // };

  auto candidate_selection = [&](Slim_Sample& slim) -> Work {
    LOG_DBG << "SELECTING CANDIDATES" << std::endl;

    Sample sample(slim);

    compute_and_set_lwr(sample);

    if (options.prescoring_by_percentage) {
      discard_bottom_x_percent(sample,
                              (1.0 - options.prescoring_threshold));
    } else {
      discard_by_accumulated_threshold(sample,
                                      options.prescoring_threshold,
                                      options.filter_min,
                                      options.filter_max);
    }

    return Work(sample);
  };

  auto thorough_placement = [&](Work& work) -> Sample {
    LOG_DBG << "BLO PLACEMENT" << std::endl;

    Sample result;
    place(work,
          chunk,
          reference_tree,
          branches,
          result,
          true,
          options,
          lookups
    );
    return result;
  };

  auto write_result = [&](Sample& sample) -> VoidToken {
    LOG_DBG << "WRITING" << std::endl;

    compute_and_set_lwr(sample);
    if (options.acc_threshold) {
      LOG_DBG << "Filtering by accumulated threshold: " << options.support_threshold << std::endl;
      discard_by_accumulated_threshold( sample,
                                        options.support_threshold,
                                        options.filter_min,
                                        options.filter_max);
    } else {
      LOG_DBG << "Filtering placements below threshold: " << options.support_threshold << std::endl;
      discard_by_support_threshold( sample,
                                    options.support_threshold,
                                    options.filter_min,
                                    options.filter_max);
    }

    // write results of current last stage aggregator node to a part file
    if (sample.size()) {
      if (chunk_num > 1) {
        outfile << ",";
      }
      outfile << sample_to_jplace_string(sample);
      // std::string part_file_name(outdir + "epa." + std::to_string(local_rank)
      //   + "." + std::to_string(chunk_num) + ".part");
      // std::ofstream part_file(part_file_name);
      // part_file << sample_to_jplace_string(sample, chunk);
      // part_names.push_back(part_file_name);
      // part_file.close();

      // // TODO for MPI, somehow ensure this code is only run on the stage foreman

      // std::ofstream status_file(status_file_name, std::ofstream::app);
      // status_file << chunk_num << ":" << chunk_size << " [";
      // for (size_t i = 0; i < part_names.size(); ++i) {
      //   status_file << part_names[i];
      //   if (i < part_names.size() - 1) status_file << ",";
      // }
      // status_file << "]" << std::endl;
      // part_names.clear();
    }

    LOG_INFO << chunk_num * chunk_size  << " Sequences done!";

    return VoidToken();
  };

  // only on one rank, only once at the end of the pipeline
  auto finalize_pipe_func = [&]() -> void {
    LOG_INFO << "Output file: " << outdir + "epa_result.jplace";
    outfile << finalize_jplace_string(invocation);
    outfile.close();
  };

  if (options.prescoring) {
    auto pipe = make_pipeline(ingestion, perloop_prehook, init_pipe_func, finalize_pipe_func)
      .push(preplacement)
      .push(candidate_selection)
      .push(thorough_placement)
      .push(write_result);

    pipe.process();
  } else {
    auto pipe = make_pipeline(ingestion, perloop_prehook, init_pipe_func, finalize_pipe_func)
      .push(thorough_placement)
      .push(write_result);

    pipe.process();
  }
}

void simple_mpi(Tree& reference_tree,
                const std::string& query_file,
                const std::string& outdir,
                const Options& options,
                const std::string& invocation)
{
  LOG_INFO << "WARNING! THIS FUNCTION IS EXPERIMENTAL!" << std::endl;

  // Timer<> flight_time;
  std::ofstream flight_file(outdir + "stat");

  std::string status_file_name(outdir + "pepa.status");
  std::ofstream trunc_status_file(status_file_name, std::ofstream::trunc);

  std::vector<std::string> part_names;

  const auto num_branches = reference_tree.nums().branches;

  // get all edges
  std::vector<pll_unode_t *> branches(num_branches);
  auto num_traversed_branches = utree_query_branches(reference_tree.tree(), &branches[0]);
  if (num_traversed_branches != num_branches) {
    throw std::runtime_error{"Traversing the utree went wrong during pipeline startup!"};
  }

  auto lookups =
    std::make_shared<Lookup_Store>(num_branches, reference_tree.partition()->states);

  // some MPI prep
  int local_rank = 0;
  int num_ranks = 1;

  MPI_COMM_RANK(MPI_COMM_WORLD, &local_rank);
  MPI_COMM_SIZE(MPI_COMM_WORLD, &num_ranks);

  LOG_DBG << "Number of ranks: " << num_ranks;

  std::vector<int> all_ranks(num_ranks);
  for (int i = 0; i < num_ranks; ++i) {
    all_ranks[i] = i;
  }

  Binary_Fasta_Reader reader(query_file);

  // how many should each rank read?
  const size_t part_size = ceil(reader.num_sequences() / static_cast<double>(num_ranks));
  LOG_DBG << "Number of sequences per rank: " << part_size;

  // read only the locally relevant part of the queries
  // ... by skipping the appropriate amount
  const size_t local_rank_seq_offset = part_size * local_rank;
  reader.skip_to_sequence( local_rank_seq_offset );
  // and limiting the reading to the given window
  reader.constrain(part_size);

  size_t num_sequences = options.chunk_size;
  Work all_work(std::make_pair(0, num_branches), std::make_pair(0, num_sequences));

  size_t chunk_num = 1;

  using Sample = Sample<Placement>;
  Sample result;
  MSA chunk;
  size_t sequences_done = 0; // not just for info output!
  while ( (num_sequences = reader.read_next(chunk, options.chunk_size) ) ) {

    assert(chunk.size() == num_sequences);

    LOG_DBG << "num_sequences: " << num_sequences << std::endl;

    const size_t seq_id_offset = sequences_done + local_rank_seq_offset;

    if (num_sequences < options.chunk_size) {
      all_work = Work(std::make_pair(0, num_branches), std::make_pair(0, num_sequences));
    }

    Sample preplace;

    LOG_DBG << "Preplacement." << std::endl;
    place(all_work,
          chunk,
          reference_tree,
          branches,
          preplace,
          false,
          options,
          lookups);

    // Candidate Selection
    LOG_DBG << "Selecting candidates." << std::endl;
    compute_and_set_lwr(preplace);

    if (options.prescoring_by_percentage) {
      discard_bottom_x_percent(preplace,
                              (1.0 - options.prescoring_threshold));
    } else {
      discard_by_accumulated_threshold( preplace,
                                        options.prescoring_threshold,
                                        options.filter_min,
                                        options.filter_max);
    }

    Work blo_work(preplace);

    Sample blo_sample;

    // BLO placement
    LOG_DBG << "BLO Placement." << std::endl;
    place(blo_work,
          chunk,
          reference_tree,
          branches,
          blo_sample,
          true,
          options,
          lookups,
          seq_id_offset);

    // Output
    compute_and_set_lwr(blo_sample);
    if (options.acc_threshold) {
      LOG_DBG << "Filtering by accumulated threshold: " << options.support_threshold << std::endl;
      discard_by_accumulated_threshold( blo_sample,
                                        options.support_threshold,
                                        options.filter_min,
                                        options.filter_max);
    } else {
      LOG_DBG << "Filtering placements below threshold: " << options.support_threshold << std::endl;
      discard_by_support_threshold( blo_sample,
                                    options.support_threshold,
                                    options.filter_min,
                                    options.filter_max);
    }

    merge(result, std::move(blo_sample));

    sequences_done += num_sequences;
    LOG_INFO << sequences_done  << " Sequences done!";
    ++chunk_num;
  }

#ifdef __MPI
  // send to output: on rank <designated_writer>
  LOG_DBG << "Gathering results on Rank " << 0;
  Timer<> dummy;
  epa_mpi_gather(result, 0, all_ranks, local_rank, dummy);
#endif //__MPI

  if (local_rank == 0) {
    // create output file
    LOG_INFO << "Output file: " << outdir + "epa_result.jplace";
    std::ofstream outfile;
    outfile.open(outdir + "epa_result.jplace");
    outfile << init_jplace_string(
      get_numbered_newick_string(reference_tree.tree()));
    outfile << sample_to_jplace_string(result);
    outfile << finalize_jplace_string(invocation);
    outfile.close();
  }

  MPI_BARRIER(MPI_COMM_WORLD);
}

<|endoftext|>


<commit_before>#include "place.hpp"

#include <fstream>
#include <chrono>
#include <omp.h>

#include "file_io.hpp"
#include "jplace_util.hpp"
#include "stringify.hpp"
#include "set_manipulators.hpp"
#include "Log.hpp"
#include "Tiny_Tree.hpp"
#include "mpihead.hpp"
#include "pll_util.hpp"
#include "epa_pll_util.hpp"
#include "Timer.hpp"
#include "Work.hpp"
#include "schedule.hpp"

#ifdef __MPI
#include "epa_mpi_util.hpp"
#endif

typedef std::unordered_map<unsigned int, std::vector<std::vector<double>>> lookupstore_t;

static void place(const Work& to_place, MSA_Stream& msa, Tree& reference_tree,
  const std::vector<pll_utree_t *>& branches, Sample& sample,
  bool do_blo, const Options& options, lookupstore_t& lookup_store)
{

#ifdef __OMP
  unsigned int num_threads = omp_get_max_threads();
#else
  unsigned int num_threads = 1;
#endif
  // split the sample structure such that the parts are thread-local
  std::vector<Sample> sample_parts(num_threads);
  std::vector<Work> work_parts;
  split(to_place, work_parts, num_threads);

  // work seperately
#ifdef __OMP
  #pragma omp parallel for schedule(dynamic)
#endif
  for (size_t i = 0; i < work_parts.size(); ++i)
  {
    for (const auto& pair : work_parts[i])
    {
      auto branch_id = pair.first;
      auto branch = Tiny_Tree(branches[branch_id], branch_id, reference_tree, do_blo, options, lookup_store[branch_id]);

      for (const auto& seq_id : pair.second)
        sample_parts[i].add_placement(seq_id, branch.place(msa[seq_id]));
    }
  }
  // merge samples back
  merge(sample, sample_parts);
}

void process(Tree& reference_tree, MSA_Stream& msa_stream, const std::string& outdir,
              const Options& options, const std::string& invocation)
{
  /* ===== COMMON DEFINITIONS ===== */
  int local_rank = 0;

#ifdef __MPI
  int world_size;
  MPI_Comm_rank(MPI_COMM_WORLD, &local_rank);
  MPI_Comm_size(MPI_COMM_WORLD, &world_size);

  lgr.dbg() << "World Size: " << world_size << std::endl;

  // shuffle available nodes to the different stages
  std::vector<std::vector<int>> schedule;
  int local_stage;
  const unsigned int num_stages = options.prescoring ? 4 : 2;

  lgr.dbg() << "Stages: " << num_stages << std::endl;

  unsigned int rebalance = 10;

  std::vector<double> init_diff = options.prescoring
    ? std::vector<double>{1000.0, 1.0, 1000.0, 1.0} : std::vector<double>{1000.0, 1.0};

  // get initial schedule
  auto init_nps = solve(num_stages, world_size, init_diff);
  assign(local_rank, init_nps, schedule, &local_stage);

  lgr.dbg() << "Schedule: ";
  for (size_t i = 0; i < schedule.size(); ++i)
  {
    lgr.dbg() << schedule[i].size() << " ";
  }
  lgr.dbg() << std::endl;

  Timer timer;

  const auto EPA_MPI_STAGE_LAST_AGGREGATE
    = options.prescoring ? EPA_MPI_STAGE_2_AGGREGATE : EPA_MPI_STAGE_1_AGGREGATE;
  const auto EPA_MPI_DEDICATED_WRITE_RANK = schedule[EPA_MPI_STAGE_LAST_AGGREGATE][0];

#endif // __MPI
  lgr << "P-EPA - Massively-Parallel Evolutionary Placement Algorithm\n";
  lgr << "\nInvocation: \n" << invocation << "\n";

  const auto chunk_size = options.chunk_size;
  lgr.dbg() << "Chunk size: " << chunk_size << std::endl;
  unsigned int num_sequences;

  const auto num_branches = reference_tree.nums().branches;

  // get all edges
  std::vector<pll_utree_t *> branches(num_branches);
  auto num_traversed_branches = utree_query_branches(reference_tree.tree(), &branches[0]);
  if(num_traversed_branches != num_branches)
    throw std::runtime_error{"Traversing the utree went wrong during pipeline startup!"};

  double lowest = 1e15;
  for (size_t i = 0; i < num_branches; ++i)
  {
    if(branches[i]->length < lowest)
      lowest = branches[i]->length;
  }

  lgr.dbg() << "smallest BL: " << lowest << std::endl;

  unsigned int chunk_num = 1;
  Sample sample;

  lookupstore_t previously_calculated_lookups;

  Work all_work(std::make_pair(0, num_branches), std::make_pair(0, chunk_size));
  Work& first_placement_work = all_work;
  Work second_placement_work; // dummy structure to be filled during operation
  std::vector<std::string> part_names; // filenames of partial results

  // while data
  // TODO this could be a stream read, such that cat msa.fasta | epa ... would work
  while ((num_sequences = msa_stream.read_next(chunk_size)) > 0)
  {
    if (num_sequences < chunk_size)
      all_work = Work(std::make_pair(0, num_branches), std::make_pair(0, num_sequences));

#ifdef __MPI
    timer.start(); // start timer of any stage
    //==============================================================
    // EPA_MPI_STAGE_1_COMPUTE === BEGIN
    //==============================================================
    if (local_stage == EPA_MPI_STAGE_1_COMPUTE)
    {

    // if previous chunk was a rebalance chunk or this is the first chunk (0 mod anything = 0)
    // ...then we need to correctly assign/reassign the workload of the first compute stage
    if ( !((chunk_num - 1) % rebalance) )
    {
      lgr.dbg() << "Assigning first stage Work" << std::endl;
      const auto& stage = schedule[EPA_MPI_STAGE_1_COMPUTE];
      std::vector<Work> parts;
      split(all_work, parts, stage.size());
      // find the stage-relative rank
      auto it = std::find(stage.begin(), stage.end(), local_rank);
      size_t stage_rank = std::distance(stage.begin(), it);
      first_placement_work = parts[stage_rank];
    }
#else
    first_placement_work = all_work;
#endif //__MPI

    place(first_placement_work, msa_stream, reference_tree, branches, sample, !options.prescoring, options, previously_calculated_lookups);

#ifdef __MPI
    // MPI: split the result and send the part to correct aggregate node
    lgr.dbg() << "Sending Stage 1 Results..." << std::endl;
    epa_mpi_split_send(sample, schedule[EPA_MPI_STAGE_1_AGGREGATE], MPI_COMM_WORLD);
    lgr.dbg() << "Stage 1 Send done!" << std::endl;

    } // endif (local_stage == EPA_MPI_STAGE_1_COMPUTE)
    //==============================================================
    // EPA_MPI_STAGE_1_COMPUTE === END
    //==============================================================

    //==============================================================
    // EPA_MPI_STAGE_1_AGGREGATE === BEGIN
    //==============================================================
    if (local_stage == EPA_MPI_STAGE_1_AGGREGATE)
    {
    // (MPI: recieve results, merge them)
    lgr.dbg() << "Recieving Stage 1 Results..." << std::endl;
    epa_mpi_recieve_merge(sample, schedule[EPA_MPI_STAGE_1_COMPUTE], MPI_COMM_WORLD);
    lgr.dbg() << "Stage 1 Recieve done!" << std::endl;
#endif // __MPI

    compute_and_set_lwr(sample);

    // if this was a prescring run, select the candidate edges
    if (options.prescoring)
    {
      if (options.prescoring_by_percentage)
        discard_bottom_x_percent(sample, (1.0 - options.prescoring_threshold));
      else
        discard_by_accumulated_threshold(sample, options.prescoring_threshold);
    }

    if(options.prescoring)
      second_placement_work = Work(sample);
#ifdef __MPI
    // if prescoring was selected, we need to send the intermediate results off to thorough placement
    if (options.prescoring)
      epa_mpi_split_send(second_placement_work, schedule[EPA_MPI_STAGE_2_COMPUTE], MPI_COMM_WORLD);
    } // endif (local_stage == EPA_MPI_STAGE_1_AGGREGATE)
    //==============================================================
    // EPA_MPI_STAGE_1_AGGREGATE === END
    //==============================================================

    //==============================================================
    // EPA_MPI_STAGE_2_COMPUTE === BEGIN
    //==============================================================
    if (local_stage == EPA_MPI_STAGE_2_COMPUTE and options.prescoring)
    {
    epa_mpi_recieve_merge(second_placement_work, schedule[EPA_MPI_STAGE_1_AGGREGATE], MPI_COMM_WORLD);
#endif // __MPI
    if (options.prescoring)
    {
      place(second_placement_work, msa_stream, reference_tree, branches, sample, true, options, previously_calculated_lookups);
    }
#ifdef __MPI
    if(options.prescoring)
      epa_mpi_split_send(sample, schedule[EPA_MPI_STAGE_2_AGGREGATE], MPI_COMM_WORLD);

    } // endif (local_stage == EPA_MPI_STAGE_2_COMPUTE)
    //==============================================================
    // EPA_MPI_STAGE_2_COMPUTE === END
    //==============================================================

    //==============================================================
    // EPA_MPI_STAGE_2_AGGREGATE === BEGIN
    //==============================================================
    if (local_stage == EPA_MPI_STAGE_LAST_AGGREGATE)
    {
      // only if this is the 4th stage do we need to get from mpi
      if (local_stage == EPA_MPI_STAGE_2_AGGREGATE)
        epa_mpi_recieve_merge(sample, schedule[EPA_MPI_STAGE_2_COMPUTE], MPI_COMM_WORLD);

#endif // __MPI
    // recompute the lwrs
    if (options.prescoring)
      compute_and_set_lwr(sample);
    // discard uninteresting placements
    if (options.acc_threshold)
      discard_by_accumulated_threshold(sample, options.support_threshold);
    else
      discard_by_support_threshold(sample, options.support_threshold);

    // write results of current last stage aggregator node to a part file
    std::string part_file_name(outdir + "epa." + std::to_string(local_rank)
      + "." + std::to_string(chunk_num) + ".part");
    std::ofstream part_file(part_file_name);
    part_file << sample_to_jplace_string(sample, msa_stream);
    part_names.push_back(part_file_name);
    part_file.close();

#ifdef __MPI
    } // endif aggregate cleanup

    //==============================================================
    // EPA_MPI_STAGE_2_AGGREGATE === END
    //==============================================================
    timer.stop(); // stop timer of any stage

    if ( !(chunk_num % rebalance) ) // time to rebalance
    {
      lgr.dbg() << "Rebalancing..." << std::endl;
      int foreman = schedule[local_stage][0];
      // Step 1: aggregate the runtime statistics, first at the lowest rank per stage
      lgr.dbg() << "aggregate the runtime statistics..." << std::endl;
      epa_mpi_gather(timer, foreman, schedule[local_stage], local_rank);
      lgr.dbg() << "Runtime aggregate done!" << std::endl;

      // Step 2: calculate average time needed per chunk for the stage
      std::vector<double> perstage_avg(num_stages);

      int color = (local_rank == foreman) ? 1 : MPI_UNDEFINED;
      MPI_Comm foreman_comm;
      MPI_Comm_split(MPI_COMM_WORLD, color, local_rank, &foreman_comm);

      if (local_rank == foreman)
      {
        double avg = timer.average();
        // Step 3: make known to all other stage representatives (mpi_allgather)
        lgr.dbg() << "Foremen allgather..." << std::endl;
        MPI_Allgather(&avg, 1, MPI_DOUBLE, &perstage_avg[0], 1, MPI_DOUBLE, foreman_comm);
        lgr.dbg() << "Foremen allgather done!" << std::endl;
        MPI_Comm_free(&foreman_comm);
      }
      MPI_BARRIER(MPI_COMM_WORLD);
      // Step 4: stage representatives forward results to all stage members
      // epa_mpi_bcast(perstage_avg, foreman, schedule[local_stage], local_rank);
      lgr.dbg() << "Broadcasting..." << std::endl;
      MPI_Comm stage_comm;
      MPI_Comm_split(MPI_COMM_WORLD, local_stage, local_rank, &stage_comm);
      MPI_Bcast(&perstage_avg[0], num_stages, MPI_DOUBLE, 0, stage_comm);
      MPI_Comm_free(&stage_comm);
      lgr.dbg() << "Broadcasting done!" << std::endl;

      // Step 5: calculate schedule on every rank, deterministically!
      to_difficulty(perstage_avg);
      auto nps = solve(num_stages, world_size, perstage_avg);
      reassign(local_rank, nps, schedule, &local_stage);
      // Step 6: re-engage pipeline with new assignments
      // compute stages should try to keep their edge assignment! affinity!
      lgr.dbg() << "Rebalancing done!" << std::endl;
    }

#endif // __MPI
    second_placement_work.clear();
    sample.clear();
    msa_stream.clear();
    lgr.dbg() << "Chunk " << chunk_num << " done!" << std::endl;
    chunk_num++;
  }

  //==============================================================
  // POST COMPUTATION
  //==============================================================
  lgr.dbg() << "Starting Post-Comp" << std::endl;
  // finally, paste all part files together
#ifdef __MPI
  MPI_BARRIER(MPI_COMM_WORLD);

  if (local_rank != EPA_MPI_DEDICATED_WRITE_RANK)
  {
    if (local_stage == EPA_MPI_STAGE_LAST_AGGREGATE)
      epa_mpi_send(part_names, EPA_MPI_DEDICATED_WRITE_RANK, MPI_COMM_WORLD);
  }
  else
  {
    for (auto rank : schedule[EPA_MPI_STAGE_LAST_AGGREGATE])
    {
      if (rank == local_rank)
        continue;
      std::vector<std::string> remote_obj;
      epa_mpi_recieve(remote_obj, rank, MPI_COMM_WORLD);
      part_names.insert(part_names.end(), remote_obj.begin(), remote_obj.end());
    }
#endif
    // create output file
    std::ofstream outfile(outdir + "epa_result.jplace");
    lgr << "\nOutput file: " << outdir + "epa_result.jplace" << std::endl;
    outfile << init_jplace_string(get_numbered_newick_string(reference_tree.tree()));
    merge_into(outfile, part_names);
    outfile << finalize_jplace_string(invocation);
    outfile.close();
#ifdef __MPI
  }
#endif
}

<commit_msg>changed timers to capture actual computational time, without waiting or communication<commit_after>#include "place.hpp"

#include <fstream>
#include <chrono>
#include <omp.h>

#include "file_io.hpp"
#include "jplace_util.hpp"
#include "stringify.hpp"
#include "set_manipulators.hpp"
#include "Log.hpp"
#include "Tiny_Tree.hpp"
#include "mpihead.hpp"
#include "pll_util.hpp"
#include "epa_pll_util.hpp"
#include "Timer.hpp"
#include "Work.hpp"
#include "schedule.hpp"

#ifdef __MPI
#include "epa_mpi_util.hpp"
#endif

typedef std::unordered_map<unsigned int, std::vector<std::vector<double>>> lookupstore_t;

static void place(const Work& to_place, MSA_Stream& msa, Tree& reference_tree,
  const std::vector<pll_utree_t *>& branches, Sample& sample,
  bool do_blo, const Options& options, lookupstore_t& lookup_store)
{

#ifdef __OMP
  unsigned int num_threads = omp_get_max_threads();
#else
  unsigned int num_threads = 1;
#endif
  // split the sample structure such that the parts are thread-local
  std::vector<Sample> sample_parts(num_threads);
  std::vector<Work> work_parts;
  split(to_place, work_parts, num_threads);

  // work seperately
#ifdef __OMP
  #pragma omp parallel for schedule(dynamic)
#endif
  for (size_t i = 0; i < work_parts.size(); ++i)
  {
    for (const auto& pair : work_parts[i])
    {
      auto branch_id = pair.first;
      auto branch = Tiny_Tree(branches[branch_id], branch_id, reference_tree, do_blo, options, lookup_store[branch_id]);

      for (const auto& seq_id : pair.second)
        sample_parts[i].add_placement(seq_id, branch.place(msa[seq_id]));
    }
  }
  // merge samples back
  merge(sample, sample_parts);
}

void process(Tree& reference_tree, MSA_Stream& msa_stream, const std::string& outdir,
              const Options& options, const std::string& invocation)
{
  /* ===== COMMON DEFINITIONS ===== */
  int local_rank = 0;

#ifdef __MPI
  int world_size;
  MPI_Comm_rank(MPI_COMM_WORLD, &local_rank);
  MPI_Comm_size(MPI_COMM_WORLD, &world_size);

  lgr.dbg() << "World Size: " << world_size << std::endl;

  // shuffle available nodes to the different stages
  std::vector<std::vector<int>> schedule;
  int local_stage;
  const unsigned int num_stages = options.prescoring ? 4 : 2;

  lgr.dbg() << "Stages: " << num_stages << std::endl;

  unsigned int rebalance = 10;

  std::vector<double> init_diff = options.prescoring
    ? std::vector<double>{1000.0, 1.0, 1000.0, 1.0} : std::vector<double>{1000.0, 1.0};

  // get initial schedule
  auto init_nps = solve(num_stages, world_size, init_diff);
  assign(local_rank, init_nps, schedule, &local_stage);

  lgr.dbg() << "Schedule: ";
  for (size_t i = 0; i < schedule.size(); ++i)
  {
    lgr.dbg() << schedule[i].size() << " ";
  }
  lgr.dbg() << std::endl;

  Timer timer;

  const auto EPA_MPI_STAGE_LAST_AGGREGATE
    = options.prescoring ? EPA_MPI_STAGE_2_AGGREGATE : EPA_MPI_STAGE_1_AGGREGATE;
  const auto EPA_MPI_DEDICATED_WRITE_RANK = schedule[EPA_MPI_STAGE_LAST_AGGREGATE][0];

#endif // __MPI
  lgr << "P-EPA - Massively-Parallel Evolutionary Placement Algorithm\n";
  lgr << "\nInvocation: \n" << invocation << "\n";

  const auto chunk_size = options.chunk_size;
  lgr.dbg() << "Chunk size: " << chunk_size << std::endl;
  unsigned int num_sequences;

  const auto num_branches = reference_tree.nums().branches;

  // get all edges
  std::vector<pll_utree_t *> branches(num_branches);
  auto num_traversed_branches = utree_query_branches(reference_tree.tree(), &branches[0]);
  if(num_traversed_branches != num_branches)
    throw std::runtime_error{"Traversing the utree went wrong during pipeline startup!"};

  double lowest = 1e15;
  for (size_t i = 0; i < num_branches; ++i)
  {
    if(branches[i]->length < lowest)
      lowest = branches[i]->length;
  }

  lgr.dbg() << "smallest BL: " << lowest << std::endl;

  unsigned int chunk_num = 1;
  Sample sample;

  lookupstore_t previously_calculated_lookups;

  Work all_work(std::make_pair(0, num_branches), std::make_pair(0, chunk_size));
  Work& first_placement_work = all_work;
  Work second_placement_work; // dummy structure to be filled during operation
  std::vector<std::string> part_names; // filenames of partial results

  // while data
  // TODO this could be a stream read, such that cat msa.fasta | epa ... would work
  while ((num_sequences = msa_stream.read_next(chunk_size)) > 0)
  {
    if (num_sequences < chunk_size)
      all_work = Work(std::make_pair(0, num_branches), std::make_pair(0, num_sequences));

#ifdef __MPI
    // timer.start(); // start timer of any stage
    //==============================================================
    // EPA_MPI_STAGE_1_COMPUTE === BEGIN
    //==============================================================
    if (local_stage == EPA_MPI_STAGE_1_COMPUTE)
    {
    // if previous chunk was a rebalance chunk or this is the first chunk (0 mod anything = 0)
    // ...then we need to correctly assign/reassign the workload of the first compute stage
    if ( !((chunk_num - 1) % rebalance) )
    {
      lgr.dbg() << "Assigning first stage Work" << std::endl;
      const auto& stage = schedule[EPA_MPI_STAGE_1_COMPUTE];
      std::vector<Work> parts;
      split(all_work, parts, stage.size());
      // find the stage-relative rank
      auto it = std::find(stage.begin(), stage.end(), local_rank);
      size_t stage_rank = std::distance(stage.begin(), it);
      first_placement_work = parts[stage_rank];
    }
    timer.start();
#else
    first_placement_work = all_work;
#endif //__MPI

    place(first_placement_work, msa_stream, reference_tree, branches, sample, !options.prescoring, options, previously_calculated_lookups);

#ifdef __MPI
    timer.stop();
    // MPI: split the result and send the part to correct aggregate node
    lgr.dbg() << "Sending Stage 1 Results..." << std::endl;
    epa_mpi_split_send(sample, schedule[EPA_MPI_STAGE_1_AGGREGATE], MPI_COMM_WORLD);
    lgr.dbg() << "Stage 1 Send done!" << std::endl;

    } // endif (local_stage == EPA_MPI_STAGE_1_COMPUTE)
    //==============================================================
    // EPA_MPI_STAGE_1_COMPUTE === END
    //==============================================================

    //==============================================================
    // EPA_MPI_STAGE_1_AGGREGATE === BEGIN
    //==============================================================
    if (local_stage == EPA_MPI_STAGE_1_AGGREGATE)
    {
    // (MPI: recieve results, merge them)
    lgr.dbg() << "Recieving Stage 1 Results..." << std::endl;
    epa_mpi_recieve_merge(sample, schedule[EPA_MPI_STAGE_1_COMPUTE], MPI_COMM_WORLD);
    lgr.dbg() << "Stage 1 Recieve done!" << std::endl;
    timer.start();
#endif // __MPI

    compute_and_set_lwr(sample);

    // if this was a prescring run, select the candidate edges
    if (options.prescoring)
    {
      if (options.prescoring_by_percentage)
        discard_bottom_x_percent(sample, (1.0 - options.prescoring_threshold));
      else
        discard_by_accumulated_threshold(sample, options.prescoring_threshold);
    }

    if(options.prescoring)
      second_placement_work = Work(sample);
#ifdef __MPI
    // if prescoring was selected, we need to send the intermediate results off to thorough placement
    if (options.prescoring)
    {
      timer.stop();
      epa_mpi_split_send(second_placement_work, schedule[EPA_MPI_STAGE_2_COMPUTE], MPI_COMM_WORLD);
    }
    } // endif (local_stage == EPA_MPI_STAGE_1_AGGREGATE)
    //==============================================================
    // EPA_MPI_STAGE_1_AGGREGATE === END
    //==============================================================

    //==============================================================
    // EPA_MPI_STAGE_2_COMPUTE === BEGIN
    //==============================================================
    if (local_stage == EPA_MPI_STAGE_2_COMPUTE and options.prescoring)
    {
    epa_mpi_recieve_merge(second_placement_work, schedule[EPA_MPI_STAGE_1_AGGREGATE], MPI_COMM_WORLD);
    timer.start();
#endif // __MPI
    if (options.prescoring)
    {
      place(second_placement_work, msa_stream, reference_tree, branches, sample, true, options, previously_calculated_lookups);
    }
#ifdef __MPI
    timer.stop();
    if(options.prescoring)
      epa_mpi_split_send(sample, schedule[EPA_MPI_STAGE_2_AGGREGATE], MPI_COMM_WORLD);

    } // endif (local_stage == EPA_MPI_STAGE_2_COMPUTE)
    //==============================================================
    // EPA_MPI_STAGE_2_COMPUTE === END
    //==============================================================

    //==============================================================
    // EPA_MPI_STAGE_2_AGGREGATE === BEGIN
    //==============================================================
    if (local_stage == EPA_MPI_STAGE_LAST_AGGREGATE)
    {
      // only if this is the 4th stage do we need to get from mpi
    if (local_stage == EPA_MPI_STAGE_2_AGGREGATE)
    {
      epa_mpi_recieve_merge(sample, schedule[EPA_MPI_STAGE_2_COMPUTE], MPI_COMM_WORLD);
      timer.start();
    }
#endif // __MPI
    // recompute the lwrs
    if (options.prescoring)
      compute_and_set_lwr(sample);
    // discard uninteresting placements
    if (options.acc_threshold)
      discard_by_accumulated_threshold(sample, options.support_threshold);
    else
      discard_by_support_threshold(sample, options.support_threshold);

    // write results of current last stage aggregator node to a part file
    std::string part_file_name(outdir + "epa." + std::to_string(local_rank)
      + "." + std::to_string(chunk_num) + ".part");
    std::ofstream part_file(part_file_name);
    part_file << sample_to_jplace_string(sample, msa_stream);
    part_names.push_back(part_file_name);
    part_file.close();

#ifdef __MPI
    timer.stop();
    } // endif aggregate cleanup

    //==============================================================
    // EPA_MPI_STAGE_2_AGGREGATE === END
    //==============================================================
    // timer.stop(); // stop timer of any stage

    if ( !(chunk_num % rebalance) ) // time to rebalance
    {
      lgr.dbg() << "Rebalancing..." << std::endl;
      int foreman = schedule[local_stage][0];
      // Step 1: aggregate the runtime statistics, first at the lowest rank per stage
      lgr.dbg() << "aggregate the runtime statistics..." << std::endl;
      epa_mpi_gather(timer, foreman, schedule[local_stage], local_rank);
      lgr.dbg() << "Runtime aggregate done!" << std::endl;

      // Step 2: calculate average time needed per chunk for the stage
      std::vector<double> perstage_avg(num_stages);

      int color = (local_rank == foreman) ? 1 : MPI_UNDEFINED;
      MPI_Comm foreman_comm;
      MPI_Comm_split(MPI_COMM_WORLD, color, local_rank, &foreman_comm);

      if (local_rank == foreman)
      {
        double avg = timer.average();
        // Step 3: make known to all other stage representatives (mpi_allgather)
        lgr.dbg() << "Foremen allgather..." << std::endl;
        MPI_Allgather(&avg, 1, MPI_DOUBLE, &perstage_avg[0], 1, MPI_DOUBLE, foreman_comm);
        lgr.dbg() << "Foremen allgather done!" << std::endl;
        MPI_Comm_free(&foreman_comm);
      }
      MPI_BARRIER(MPI_COMM_WORLD);
      // Step 4: stage representatives forward results to all stage members
      // epa_mpi_bcast(perstage_avg, foreman, schedule[local_stage], local_rank);
      lgr.dbg() << "Broadcasting..." << std::endl;
      MPI_Comm stage_comm;
      MPI_Comm_split(MPI_COMM_WORLD, local_stage, local_rank, &stage_comm);
      MPI_Bcast(&perstage_avg[0], num_stages, MPI_DOUBLE, 0, stage_comm);
      MPI_Comm_free(&stage_comm);
      lgr.dbg() << "Broadcasting done!" << std::endl;

      lgr.dbg() << "perstage average:";
      for (size_t i = 0; i < perstage_avg.size(); ++i)
      {
        lgr.dbg() << " " << perstage_avg[i];
      }
      lgr.dbg() << std::endl;

      // Step 5: calculate schedule on every rank, deterministically!
      to_difficulty(perstage_avg);

      lgr.dbg() << "perstage difficulty:";
      for (size_t i = 0; i < perstage_avg.size(); ++i)
      {
        lgr.dbg() << " " << perstage_avg[i];
      }
      lgr.dbg() << std::endl;

      auto nps = solve(num_stages, world_size, perstage_avg);
      reassign(local_rank, nps, schedule, &local_stage);
      // Step 6: re-engage pipeline with new assignments
      lgr.dbg() << "New Schedule:";
      for (size_t i = 0; i < nps.size(); ++i)
        lgr.dbg() << " " << nps[i];
      lgr.dbg() << std::endl;
      // compute stages should try to keep their edge assignment! affinity!
      lgr.dbg() << "Rebalancing done!" << std::endl;
    }

#endif // __MPI
    second_placement_work.clear();
    sample.clear();
    msa_stream.clear();
    lgr.dbg() << "Chunk " << chunk_num << " done!" << std::endl;
    chunk_num++;
  }

  //==============================================================
  // POST COMPUTATION
  //==============================================================
  lgr.dbg() << "Starting Post-Comp" << std::endl;
  // finally, paste all part files together
#ifdef __MPI
  MPI_BARRIER(MPI_COMM_WORLD);

  if (local_rank != EPA_MPI_DEDICATED_WRITE_RANK)
  {
    if (local_stage == EPA_MPI_STAGE_LAST_AGGREGATE)
      epa_mpi_send(part_names, EPA_MPI_DEDICATED_WRITE_RANK, MPI_COMM_WORLD);
  }
  else
  {
    for (auto rank : schedule[EPA_MPI_STAGE_LAST_AGGREGATE])
    {
      if (rank == local_rank)
        continue;
      std::vector<std::string> remote_obj;
      epa_mpi_recieve(remote_obj, rank, MPI_COMM_WORLD);
      part_names.insert(part_names.end(), remote_obj.begin(), remote_obj.end());
    }
#endif
    // create output file
    std::ofstream outfile(outdir + "epa_result.jplace");
    lgr << "\nOutput file: " << outdir + "epa_result.jplace" << std::endl;
    outfile << init_jplace_string(get_numbered_newick_string(reference_tree.tree()));
    merge_into(outfile, part_names);
    outfile << finalize_jplace_string(invocation);
    outfile.close();
#ifdef __MPI
  }
#endif
}

<|endoftext|>


<commit_before>/*
 * Copyright (c) 2014 New Designs Unlimited, LLC
 * Opensource Automated License Plate Recognition [http://www.openalpr.com]
 *
 * This file is part of OpenAlpr.
 *
 * OpenAlpr is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License
 * version 3 as published by the Free Software Foundation
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "detectormorph.h"

using namespace cv;
using namespace std;

namespace alpr {

  DetectorMorph::DetectorMorph(Config* config) : Detector(config) {

    this->loaded = true;
  }

  DetectorMorph::~DetectorMorph() {
  }

  vector<PlateRegion> DetectorMorph::detect(Mat frame, std::vector<cv::Rect> regionsOfInterest) {

    Mat frame_gray;
    cvtColor(frame, frame_gray, CV_BGR2GRAY);

    vector<PlateRegion> detectedRegions;
    for (int i = 0; i < regionsOfInterest.size(); i++) {
      Mat img_open, img_result;
      Mat element = getStructuringElement(MORPH_RECT, Size(30, 4));
      morphologyEx(frame_gray, img_open, CV_MOP_OPEN, element, cv::Point(-1, -1));

      img_result = frame_gray - img_open;

      if (config->debugDetector && config->debugShowImages) {
        imshow("Opening", img_result);
      }

      //threshold image using otsu thresholding
      Mat img_threshold, img_open2;
      threshold(img_result, img_threshold, 0, 255, CV_THRESH_OTSU + CV_THRESH_BINARY);

      if (config->debugDetector && config->debugShowImages) {
        imshow("Threshold Detector", img_threshold);
      }

      Mat circularElement = getStructuringElement(cv::MORPH_ELLIPSE, Size(5, 5));
      morphologyEx(img_threshold, img_open2, CV_MOP_OPEN, circularElement, cv::Point(-1, -1));
      Mat rectElement = getStructuringElement(cv::MORPH_RECT, Size(20, 4));
      morphologyEx(img_open2, img_threshold, CV_MOP_CLOSE, rectElement, cv::Point(-1, -1));

      if (config->debugDetector && config->debugShowImages) {
        imshow("Close", img_threshold);
        waitKey(0);
      }

      //Find contours of possibles plates
      vector< vector< Point> > contours;
      findContours(img_threshold,
              contours, // a vector of contours
              CV_RETR_EXTERNAL, // retrieve the external contours
              CV_CHAIN_APPROX_NONE); // all pixels of each contours

      //Start to iterate to each contour founded
      vector<vector<Point> >::iterator itc = contours.begin();
      vector<RotatedRect> rects;

      //Remove patch that are no inside limits of aspect ratio and area.
      while (itc != contours.end()) {
        //Create bounding rect of object
        RotatedRect mr = minAreaRect(Mat(*itc));
        if (!CheckSizes(mr))
          itc = contours.erase(itc);
        else {
          ++itc;
          if (itc == contours.end()) continue;
          rects.push_back(mr);
        }
      }

      //Now prunning based on checking all candidate plates for a min/max number of blobs
      Mat img_crop, img_crop_b;
      vector< vector< Point> > plateBlobs;
      for (int i = 0; i < rects.size(); i++) {
        RotatedRect PlateRect = rects[i];
        Size rect_size = PlateRect.size;

        //Crop area around candidate plate
        getRectSubPix(frame_gray, rect_size, PlateRect.center, img_crop);
        //Thresholding
        threshold(img_crop, img_crop_b, 0, 255, cv::THRESH_BINARY_INV + cv::THRESH_OTSU);

        findContours(img_crop_b,
                plateBlobs, // a vector of contours
                CV_RETR_EXTERNAL, // retrieve the external contours
                CV_CHAIN_APPROX_NONE); // all pixels of each contours

        int numBlobs = plateBlobs.size();

        //If too much or too little might not be a true plate
        if (numBlobs < 3 || numBlobs > 20) continue;

        PlateRegion PlateReg;

        PlateReg.rect = PlateRect.boundingRect();

        detectedRegions.push_back(PlateReg);

      }

    }

    return detectedRegions;
  }

  bool DetectorMorph::CheckSizes(RotatedRect& mr) {

    float error = 1.2;

    float aspect = config->plateWidthMM / config->plateHeightMM;
    //Set a min and max area. All other patchs are discarded
    int min = 10 * aspect * 10; // minimum area
    int max = 100 * aspect * 100; // maximum area
    //Get only patchs that match to a respect ratio.
    float rmin = 3.0;
    float rmax = 7.0;

    int area = mr.size.height * mr.size.width;
    float r = (float) mr.size.width / (float) mr.size.height;
    if (r < 1)
      r = (float) mr.size.height / (float) mr.size.width;

    if ((area < min || area > max) || (r < rmin || r > rmax) ||
            (mr.size.width < 20 || mr.size.width > 300))
      return false;
    else
      return true;

  }

}
<commit_msg>Fixed a crash caused by detected regions that were beyond the beyond of the image<commit_after>/*
 * Copyright (c) 2014 New Designs Unlimited, LLC
 * Opensource Automated License Plate Recognition [http://www.openalpr.com]
 *
 * This file is part of OpenAlpr.
 *
 * OpenAlpr is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License
 * version 3 as published by the Free Software Foundation
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "detectormorph.h"

using namespace cv;
using namespace std;

namespace alpr {

  DetectorMorph::DetectorMorph(Config* config) : Detector(config) {

    this->loaded = true;
  }

  DetectorMorph::~DetectorMorph() {
  }

  vector<PlateRegion> DetectorMorph::detect(Mat frame, std::vector<cv::Rect> regionsOfInterest) {

    Mat frame_gray;
    cvtColor(frame, frame_gray, CV_BGR2GRAY);

    vector<PlateRegion> detectedRegions;
    for (int i = 0; i < regionsOfInterest.size(); i++) {
      Mat img_open, img_result;
      Mat element = getStructuringElement(MORPH_RECT, Size(30, 4));
      morphologyEx(frame_gray, img_open, CV_MOP_OPEN, element, cv::Point(-1, -1));

      img_result = frame_gray - img_open;

      if (config->debugDetector && config->debugShowImages) {
        imshow("Opening", img_result);
      }

      //threshold image using otsu thresholding
      Mat img_threshold, img_open2;
      threshold(img_result, img_threshold, 0, 255, CV_THRESH_OTSU + CV_THRESH_BINARY);

      if (config->debugDetector && config->debugShowImages) {
        imshow("Threshold Detector", img_threshold);
      }

      Mat circularElement = getStructuringElement(cv::MORPH_ELLIPSE, Size(5, 5));
      morphologyEx(img_threshold, img_open2, CV_MOP_OPEN, circularElement, cv::Point(-1, -1));
      Mat rectElement = getStructuringElement(cv::MORPH_RECT, Size(20, 4));
      morphologyEx(img_open2, img_threshold, CV_MOP_CLOSE, rectElement, cv::Point(-1, -1));

      if (config->debugDetector && config->debugShowImages) {
        imshow("Close", img_threshold);
        waitKey(0);
      }

      //Find contours of possibles plates
      vector< vector< Point> > contours;
      findContours(img_threshold,
              contours, // a vector of contours
              CV_RETR_EXTERNAL, // retrieve the external contours
              CV_CHAIN_APPROX_NONE); // all pixels of each contours

      //Start to iterate to each contour founded
      vector<vector<Point> >::iterator itc = contours.begin();
      vector<RotatedRect> rects;

      //Remove patch that are no inside limits of aspect ratio and area.
      while (itc != contours.end()) {
        //Create bounding rect of object
        RotatedRect mr = minAreaRect(Mat(*itc));
        if (!CheckSizes(mr))
          itc = contours.erase(itc);
        else {
          ++itc;
          if (itc == contours.end()) continue;
          rects.push_back(mr);
        }
      }

      //Now prunning based on checking all candidate plates for a min/max number of blobs
      Mat img_crop, img_crop_b;
      vector< vector< Point> > plateBlobs;
      for (int i = 0; i < rects.size(); i++) {
        RotatedRect PlateRect = rects[i];
        Size rect_size = PlateRect.size;

        //Crop area around candidate plate
        getRectSubPix(frame_gray, rect_size, PlateRect.center, img_crop);
        //Thresholding
        threshold(img_crop, img_crop_b, 0, 255, cv::THRESH_BINARY_INV + cv::THRESH_OTSU);

        findContours(img_crop_b,
                plateBlobs, // a vector of contours
                CV_RETR_EXTERNAL, // retrieve the external contours
                CV_CHAIN_APPROX_NONE); // all pixels of each contours

        int numBlobs = plateBlobs.size();

        //If too much or too little might not be a true plate
        if (numBlobs < 3 || numBlobs > 20) continue;

        PlateRegion PlateReg;

        PlateReg.rect = PlateRect.boundingRect();

        if (PlateReg.rect.x < 0)
          PlateReg.rect.x = 0;
        if (PlateReg.rect.y < 0)
          PlateReg.rect.y = 0;
        if (PlateReg.rect.x + PlateReg.rect.width > frame.cols)
          PlateReg.rect.width = frame.cols - PlateReg.rect.x;
        if (PlateReg.rect.y + PlateReg.rect.height > frame.rows)
          PlateReg.rect.height = frame.rows - PlateReg.rect.y;

        detectedRegions.push_back(PlateReg);

      }

    }

    return detectedRegions;
  }

  bool DetectorMorph::CheckSizes(RotatedRect& mr) {

    float error = 1.2;

    float aspect = config->plateWidthMM / config->plateHeightMM;
    //Set a min and max area. All other patchs are discarded
    int min = 10 * aspect * 10; // minimum area
    int max = 100 * aspect * 100; // maximum area
    //Get only patchs that match to a respect ratio.
    float rmin = 3.0;
    float rmax = 7.0;

    int area = mr.size.height * mr.size.width;
    float r = (float) mr.size.width / (float) mr.size.height;
    if (r < 1)
      r = (float) mr.size.height / (float) mr.size.width;

    if ((area < min || area > max) || (r < rmin || r > rmax) ||
            (mr.size.width < 20 || mr.size.width > 300))
      return false;
    else
      return true;

  }

}
<|endoftext|>


<commit_before>/*-------------FDTD.cpp-------------------------------------------------------//
*
*              Finite Difference Time Domain
*
* Purpose: To replicate the results of our invisible lense raytracer with
*          FDTD. Woo!
*
*   Notes: Most of this is coming from the following link:
*             http://www.eecs.wsu.edu/~schneidj/ufdtd/chap3.pdf
*
*-----------------------------------------------------------------------------*/

#include <iostream>
#include <vector>
#include <cmath>
#include <fstream>

static const size_t space = 200;

struct Loss{
    std::vector <double> EzH = std::vector<double>(space * space, 0),
                         EzE = std::vector<double>(space * space, 0),
                         HyE = std::vector<double>(space * space, 0),
                         HyH = std::vector<double>(space * space, 0),
                         HxE = std::vector<double>(space * space, 0),
                         HxH = std::vector<double>(space * space, 0);
};

struct Field{
    std::vector <double> Hx = std::vector<double>(space * space, 0),
                         Hy = std::vector<double>(space * space, 0),
                         Ez = std::vector<double>(space * space, 0);
};

#define EzH(i, j) EzH[(i) + (j) *  space]
#define EzE(i, j) EzE[(i) + (j) *  space]
#define HyH(i, j) HyH[(i) + (j) *  space]
#define HyE(i, j) HyE[(i) + (j) *  space]
#define HxH(i, j) HxH[(i) + (j) *  space]
#define HxE(i, j) HxE[(i) + (j) *  space]
#define Hx(i, j) Hx[(i) + (j) *  space]
#define Hy(i, j) Hy[(i) + (j) *  space]
#define Ez(i, j) Ez[(i) + (j) *  space]

void FDTD(Field EM,
          const int final_time, const double eps, const int space, Loss lass,
          std::ofstream& output);

/*----------------------------------------------------------------------------//
* MAIN
*-----------------------------------------------------------------------------*/

int main(){

    // defines output
    std::ofstream output("FDTD.dat", std::ofstream::out);

    int space = 200, final_time = 500;
    double eps = 377.0;

    Loss lass;

    // define initial E and H fields
    // std::vector<double> Ez(space, 0.0), Hy(space, 0.0);
    Field EM;

    FDTD(EM, final_time, eps, space, lass, output);

}

/*----------------------------------------------------------------------------//
* SUBROUTINES
*-----------------------------------------------------------------------------*/

// This is the function we writs the bulk of the code in
void FDTD(Field EM,
          const int final_time, const double eps, const int space, Loss lass,
          std::ofstream& output){

    // For magnetic field:
    // double offset = 0.00005;

    // for electric field:
    double offset = 0.05;
    double loss = 0.0;

    // Relative permittivity
    for (int dx = 0; dx < space; dx++){
        for (int dy = 0; dy < space; dy++){
            if (dx > 100 && dx < 150){
                lass.EzH(dx, dy) =  eps / 9.0 /(1.0 - loss);
                lass.EzE(dx, dy) = (1.0 - loss) / (1.0 + loss);
                lass.HyH(dx, dy) = (1.0 - loss) / (1.0 + loss);
                lass.HyE(dx, dy) = (1.0 / eps) / (1.0 + loss);
                lass.HxE(dx, dy) = (1.0 / eps) / (1.0 + loss);
                lass.HxH(dx, dy) = (1.0 - loss) / (1.0 + loss);
            }
            else{
                lass.EzH(dx, dy) =  eps;
                lass.EzE(dx, dy) = 1.0;
                lass.HyH(dx, dy) = 1.0;
                lass.HyE(dx, dy) = (1.0 / eps);
                lass.HxE(dx, dy) = (1.0 / eps);
                lass.HxH(dx, dy) = 1.0;
            }
        }
    }

    // Time looping
    for (int t = 0; t < final_time; t++){

        // Linking the final two elements for an ABC
        // Hy[space - 1] = Hy[space - 2];

        // update magnetic field, y direction
        for (int dx = 0; dx < space - 1; dx++){
            for (int dy = 0; dy < space; dy++){
               EM.Hy(dx,dy) = lass.HyH(dx,dy) * EM.Hy(dx,dy)
                           + lass.HyE(dx,dy) * (EM.Ez(dx + 1,dy)
                                                - EM.Ez(dx,dy));
            }
        }

        // update magnetic field, x direction
        for (int dx = 0; dx < space; dx++){
            for (int dy = 0; dy < space - 1; dy++){
               EM.Hx(dx,dy) = lass.HxH(dx,dy) * EM.Hx(dx, dy)
                           + lass.HxE(dx,dy) * (EM.Ez(dx,dy + 1)
                                                - EM.Ez(dx,dy));
            }
        }

        // Correction to the H field for the TFSF boundary
        // Hy[49] -= exp(-(t - 40.) * (t - 40.)/100.0) / eps;
        // Hy[49] -= sin((t-10.0)*0.2)*0.0005;

        // Linking the first two elements in the electric field
        EM.Ez[0] = EM.Ez[1];
        // Ez[space - 1] = Ez[space - 2];

        // update electric field
        for (int dx = 1; dx < space - 1; dx++){
            for (int dy = 1; dy < space - 1; dy++){
               EM.Ez(dx,dy) = lass.EzE(dx,dy) * EM.Ez(dx,dy)
                           + lass.EzH[dx] * (EM.Hy(dx, dy)
                                             - EM.Hy(dx - 1, dy)
                                             - EM.Hx(dx,dy)
                                             - EM.Hx(dx, dy - 1));
            }
        }

        // set src for next step
        EM.Ez[0] += exp(-((t + 1 - 40.) * (t + 1 - 40.))/100.0);
        // EM.Ez[0] = 0;

        // Ez[50] += sin((t - 10.0 + 1)*0.2)*0.0005;
/*
        if (t > 0){
            Ez[50] = sin(0.1 * t) / (0.1 * t);
        }
        else{
            Ez[50] = 1;
        }
*/
        if (t % 50 == 0){
            for (int dx = 0; dx < space; dx = dx + 5){
                for (int dy = 0; dy < space; dy = dy + 5){
                    output << t << '\t' << dx <<'\t' << dy
                           << EM.Ez(dx, dy) << '\n';
                    //output << Ez(dx,dy) + (t * offset) << '\n';
                    //output << Hy[dx] + (t * offset) << '\n';
                }
            }

            output << '\n' << '\n';
        }

    }
}
<commit_msg>adding a few changes, submitting for Kramsfasel.<commit_after>/*-------------FDTD.cpp-------------------------------------------------------//
*
*              Finite Difference Time Domain
*
* Purpose: To replicate the results of our invisible lense raytracer with
*          FDTD. Woo!
*
*   Notes: Most of this is coming from the following link:
*             http://www.eecs.wsu.edu/~schneidj/ufdtd/chap3.pdf
*
*-----------------------------------------------------------------------------*/

#include <iostream>
#include <vector>
#include <cmath>
#include <fstream>

static const size_t space = 200;

struct Loss{
    std::vector <double> EzH = std::vector<double>(space * space, 0),
                         EzE = std::vector<double>(space * space, 0),
                         HyE = std::vector<double>(space * space, 0),
                         HyH = std::vector<double>(space * space, 0),
                         HxE = std::vector<double>(space * space, 0),
                         HxH = std::vector<double>(space * space, 0);
};

struct Field{
    std::vector <double> Hx = std::vector<double>(space * space, 0),
                         Hy = std::vector<double>(space * space, 0),
                         Ez = std::vector<double>(space * space, 0);
};

#define EzH(i, j) EzH[(i) + (j) *  space]
#define EzE(i, j) EzE[(i) + (j) *  space]
#define HyH(i, j) HyH[(i) + (j) *  space]
#define HyE(i, j) HyE[(i) + (j) *  space]
#define HxH(i, j) HxH[(i) + (j) *  space]
#define HxE(i, j) HxE[(i) + (j) *  space]
#define Hx(i, j) Hx[(i) + (j) *  space]
#define Hy(i, j) Hy[(i) + (j) *  space]
#define Ez(i, j) Ez[(i) + (j) *  space]

void FDTD(Field EM,
          const int final_time, const double eps, const int space, Loss lass,
          std::ofstream& output);

/*----------------------------------------------------------------------------//
* MAIN
*-----------------------------------------------------------------------------*/

int main(){

    // defines output
    std::ofstream output("FDTD.dat", std::ofstream::out);

    int space = 200, final_time = 100;
    double eps = 377.0;

    Loss lass;

    // define initial E and H fields
    // std::vector<double> Ez(space, 0.0), Hy(space, 0.0);
    Field EM;

    FDTD(EM, final_time, eps, space, lass, output);

}

/*----------------------------------------------------------------------------//
* SUBROUTINES
*-----------------------------------------------------------------------------*/

// This is the function we writs the bulk of the code in
void FDTD(Field EM,
          const int final_time, const double eps, const int space, Loss lass,
          std::ofstream& output){

    // For magnetic field:
    // double offset = 0.00005;

    // for electric field:
    double offset = 0.05;
    double loss = 0.0;

    // Relative permittivity
    for (int dx = 0; dx < space; dx++){
        for (int dy = 0; dy < space; dy++){
            if (dx > 100 && dx < 150){
                lass.EzH(dx, dy) =  eps / 9.0 /(1.0 - loss);
                lass.EzE(dx, dy) = (1.0 - loss) / (1.0 + loss);
                lass.HyH(dx, dy) = (1.0 - loss) / (1.0 + loss);
                lass.HyE(dx, dy) = (1.0 / eps) / (1.0 + loss);
                lass.HxE(dx, dy) = (1.0 / eps) / (1.0 + loss);
                lass.HxH(dx, dy) = (1.0 - loss) / (1.0 + loss);
            }
            else{
                lass.EzH(dx, dy) =  eps;
                lass.EzE(dx, dy) = 1.0;
                lass.HyH(dx, dy) = 1.0;
                lass.HyE(dx, dy) = (1.0 / eps);
                lass.HxE(dx, dy) = (1.0 / eps);
                lass.HxH(dx, dy) = 1.0;
            }
        }
    }

    // Time looping
    for (int t = 0; t < final_time; t++){

        // Linking the final two elements for an ABC
        // Hy[space - 1] = Hy[space - 2];

        // update magnetic field, y direction
        for (int dx = 0; dx < space - 1; dx++){
            for (int dy = 0; dy < space; dy++){
               EM.Hy(dx,dy) = lass.HyH(dx,dy) * EM.Hy(dx,dy)
                           + lass.HyE(dx,dy) * (EM.Ez(dx + 1,dy)
                                                - EM.Ez(dx,dy));
            }
        }

        // update magnetic field, x direction
        for (int dx = 0; dx < space; dx++){
            for (int dy = 0; dy < space - 1; dy++){
               EM.Hx(dx,dy) = lass.HxH(dx,dy) * EM.Hx(dx, dy)
                           + lass.HxE(dx,dy) * (EM.Ez(dx,dy + 1)
                                                - EM.Ez(dx,dy));
            }
        }

        // Correction to the H field for the TFSF boundary
        // Hy[49] -= exp(-(t - 40.) * (t - 40.)/100.0) / eps;
        // Hy[49] -= sin((t-10.0)*0.2)*0.0005;

        // Linking the first two elements in the electric field
        EM.Ez[0] = EM.Ez[1];
        // Ez[space - 1] = Ez[space - 2];

        // update electric field
        for (int dx = 1; dx < space - 1; dx++){
            for (int dy = 1; dy < space - 1; dy++){
               EM.Ez(dx,dy) = lass.EzE(dx,dy) * EM.Ez(dx,dy)
                           + lass.EzH[dx] * (EM.Hy(dx, dy)
                                             - EM.Hy(dx - 1, dy)
                                             - EM.Hx(dx,dy)
                                             - EM.Hx(dx, dy - 1));
            }
        }

        // set src for next step
        EM.Ez[50] += exp(-((t + 1 - 40.) * (t + 1 - 40.))/100.0);
        // EM.Ez[0] = 0;

        // Ez[50] += sin((t - 10.0 + 1)*0.2)*0.0005;
/*
        if (t > 0){
            Ez[50] = sin(0.1 * t) / (0.1 * t);
        }
        else{
            Ez[50] = 1;
        }
*/
        if (t % 50 == 0){
            for (int dx = 0; dx < space; dx = dx + 5){
                for (int dy = 0; dy < space; dy = dy + 5){
                    output << t << '\t' << dx <<'\t' << dy << '\t'
                           << EM.Hx(dx, dy) << '\n';
                    //output << Ez(dx,dy) + (t * offset) << '\n';
                    //output << Hy[dx] + (t * offset) << '\n';
                }
            }

            output << '\n' << '\n';
        }

    }
}
<|endoftext|>


<commit_before>
#include "include/rdata.h"

ReturnData::ReturnData()
{
    setDefaults();
}

ReturnData::ReturnData(const UserData *udata)
{
    setDefaults();

    initFields(udata);
}

void ReturnData::setDefaults()
{
    ts = xdot = dxdotdp = dydx = dydp = J = z = sigmaz = sz = ssigmaz = rz = srz = s2rz = x = sx = y = sigmay = NULL;
    sy = ssigmay = numsteps = numstepsB = numrhsevals = numrhsevalsB = order = llh = chi2 = sllh = s2llh = NULL;
    numerrtestfails = numnonlinsolvconvfails = NULL;
    status = NULL;

    freeFieldsOnDestruction = true;
}

ReturnData::~ReturnData()
{
    if(!freeFieldsOnDestruction)
        return;

    if(ts) delete[] ts;
    if(xdot) delete[] xdot;
    if(dxdotdp) delete[] dxdotdp;
    if(dydx) delete[] dydx;
    if(dydp) delete[] dydp;
    if(J) delete[] J;
    if(z) delete[] z;
    if(sigmaz) delete[] sigmaz;
    if(sz) delete[] sz;
    if(ssigmaz) delete[] ssigmaz;
    if(rz) delete[] rz;
    if(srz) delete[] srz;
    if(s2rz) delete[] s2rz;
    if(x) delete[] x;
    if(sx) delete[] sx;
    if(y) delete[] y;
    if(sigmay) delete[] sigmay;
    if(sy) delete[] sy;
    if(ssigmay) delete[] ssigmay;
    if(numsteps) delete[] numsteps;
    if(numrhsevals) delete[] numrhsevals;
    if(numerrtestfails) delete [] numerrtestfails;
    if(numnonlinsolvconvfails) delete [] numnonlinsolvconvfails;
    if(order) delete[] order;
    if(numstepsB) delete[] numstepsB;
    if(numrhsevalsB) delete[] numrhsevalsB;
    if(numerrtestfailsB) delete [] numerrtestfailsB;
    if(numnonlinsolvconvfailsB) delete [] numnonlinsolvconvfailsB;
    if(llh) delete[] llh;
    if(sllh) delete[] sllh;
    if(s2llh) delete[] s2llh;
    if(chi2) delete[] chi2;
    if(status) delete[] status;
}

void ReturnData::initFields(const UserData *udata)
{
    if(udata) {
        initField1(&status, "status", 1);
        initField1(&ts, "t", udata->nt);
        initField1(&llh, "llh", 1);
        initField1(&chi2, "chi2", 1);
        initField2(&numsteps, "numsteps",udata->nt, 1);
        initField2(&numrhsevals, "numrhsevals", udata->nt, 1);
        initField2(&numerrtestfails, "numerrtestfails",udata->nt, 1);
        initField2(&numnonlinsolvconvfails, "numnonlinsolvconvfails", udata->nt, 1);
        initField2(&order, "order", udata->nt,1);

        if((udata->nz>0) & (udata->ne>0)){
            initField2(&z, "z", udata->nmaxevent,udata->nz);
            initField2(&rz, "rz", udata->nmaxevent,udata->nz);
            initField2(&sigmaz, "sigmaz", udata->nmaxevent,udata->nz);
        }
        if(udata->nx>0) {
            initField2(&x, "x", udata->nt,udata->nx);
            initField2(&xdot, "xdot", 1,udata->nx);
            initField2(&J, "J", udata->nx,udata->nx);
        }
        if(udata->ny>0) {
            initField2(&y, "y", udata->nt,udata->ny);
            initField2(&sigmay, "sigmay", udata->nt,udata->ny);
            if (udata->sensi_meth == AMICI_SENSI_SS) {
                initField2(&dydp, "dydp", udata->ny,udata->nplist);
                initField2(&dydx, "dydx", udata->ny,udata->nx);
                initField2(&dxdotdp, "dxdotdp", udata->nx,udata->nplist);
            }
        }
        if(udata->sensi >= AMICI_SENSI_ORDER_FIRST) {
            initField2(&sllh, "sllh", udata->nplist,1);

            if (udata->sensi_meth == AMICI_SENSI_FSA) {
                initField3(&sx, "sx", udata->nt,udata->nx,udata->nplist);
                if(udata->ny>0) {
                    initField3(&sy, "sy", udata->nt,udata->ny,udata->nplist);
                    initField3(&ssigmay, "ssigmay", udata->nt,udata->ny,udata->nplist);
                }
                if((udata->nz>0) & (udata->ne>0)){
                    initField3(&srz, "srz", udata->nmaxevent,udata->nz,udata->nplist);
                    if(udata->sensi >= AMICI_SENSI_ORDER_SECOND){
                        initField4(&s2rz, "s2rz", udata->nmaxevent,udata->nz,udata->nplist,udata->nplist);
                    }
                    initField3(&sz, "sz", udata->nmaxevent,udata->nz,udata->nplist);
                    initField3(&ssigmaz, "ssigmaz", udata->nmaxevent,udata->nz,udata->nplist);
                }
            }

            if (udata->sensi_meth == AMICI_SENSI_ASA) {
                if(udata->ny>0) {
                    initField3(&ssigmay, "ssigmay", udata->nt,udata->ny,udata->nplist);
                }
                if((udata->nz>0) & (udata->ne>0)){
                    initField3(&ssigmaz, "ssigmaz", udata->nmaxevent,udata->nz,udata->nplist);
                }
                initField2(&numstepsB, "numstepsB", udata->nt, 1);
                initField2(&numrhsevalsB, "numrhsevalsB", udata->nt, 1);
                initField2(&numerrtestfailsB, "numerrtestfailsB",udata->nt, 1);
                initField2(&numnonlinsolvconvfailsB, "numnonlinsolvconvfailsB", udata->nt, 1);
            }

            if(udata->sensi >= AMICI_SENSI_ORDER_SECOND) {
                initField2(&s2llh, "s2llh", udata->ng-1,udata->nplist);
            }
        }
    }
}

void ReturnData::initField1(double **fieldPointer, const char *fieldName, int dim)
{
    *fieldPointer = new double[dim]();
}

void ReturnData::initField2(double **fieldPointer, const char *fieldName, int dim1, int dim2)
{
    *fieldPointer = new double[(dim1) * (dim2)]();

}

void ReturnData::initField3(double **fieldPointer, const char *fieldName, int dim1, int dim2, int dim3)
{
    *fieldPointer = new double[(dim1) * (dim2) * (dim3)]();
}

void ReturnData::initField4(double **fieldPointer, const char *fieldName, int dim1, int dim2, int dim3, int dim4)
{
    *fieldPointer = new double[(dim1) * (dim2) * (dim3) * (dim4)]();
}
<commit_msg>fix rdata initialisation<commit_after>
#include "include/rdata.h"

ReturnData::ReturnData()
{
    setDefaults();
}

ReturnData::ReturnData(const UserData *udata)
{
    setDefaults();

    initFields(udata);
}

void ReturnData::setDefaults()
{
    ts = xdot = dxdotdp = dydx = dydp = J = z = sigmaz = sz = ssigmaz = rz = srz = s2rz = x = sx = y = sigmay = NULL;
    sy = ssigmay = numsteps = numstepsB = numrhsevals = numrhsevalsB = order = llh = chi2 = sllh = s2llh = NULL;
    numerrtestfails = numnonlinsolvconvfails = numerrtestfailsB = numnonlinsolvconvfailsB = NULL;
    status = NULL;

    freeFieldsOnDestruction = true;
}

ReturnData::~ReturnData()
{
    if(!freeFieldsOnDestruction)
        return;

    if(ts) delete[] ts;
    if(xdot) delete[] xdot;
    if(dxdotdp) delete[] dxdotdp;
    if(dydx) delete[] dydx;
    if(dydp) delete[] dydp;
    if(J) delete[] J;
    if(z) delete[] z;
    if(sigmaz) delete[] sigmaz;
    if(sz) delete[] sz;
    if(ssigmaz) delete[] ssigmaz;
    if(rz) delete[] rz;
    if(srz) delete[] srz;
    if(s2rz) delete[] s2rz;
    if(x) delete[] x;
    if(sx) delete[] sx;
    if(y) delete[] y;
    if(sigmay) delete[] sigmay;
    if(sy) delete[] sy;
    if(ssigmay) delete[] ssigmay;
    if(numsteps) delete[] numsteps;
    if(numrhsevals) delete[] numrhsevals;
    if(numerrtestfails) delete [] numerrtestfails;
    if(numnonlinsolvconvfails) delete [] numnonlinsolvconvfails;
    if(order) delete[] order;
    if(numstepsB) delete[] numstepsB;
    if(numrhsevalsB) delete[] numrhsevalsB;
    if(numerrtestfailsB) delete [] numerrtestfailsB;
    if(numnonlinsolvconvfailsB) delete [] numnonlinsolvconvfailsB;
    if(llh) delete[] llh;
    if(sllh) delete[] sllh;
    if(s2llh) delete[] s2llh;
    if(chi2) delete[] chi2;
    if(status) delete[] status;
}

void ReturnData::initFields(const UserData *udata)
{
    if(udata) {
        initField1(&status, "status", 1);
        initField1(&ts, "t", udata->nt);
        initField1(&llh, "llh", 1);
        initField1(&chi2, "chi2", 1);
        initField2(&numsteps, "numsteps",udata->nt, 1);
        initField2(&numrhsevals, "numrhsevals", udata->nt, 1);
        initField2(&numerrtestfails, "numerrtestfails",udata->nt, 1);
        initField2(&numnonlinsolvconvfails, "numnonlinsolvconvfails", udata->nt, 1);
        initField2(&order, "order", udata->nt,1);

        if((udata->nz>0) & (udata->ne>0)){
            initField2(&z, "z", udata->nmaxevent,udata->nz);
            initField2(&rz, "rz", udata->nmaxevent,udata->nz);
            initField2(&sigmaz, "sigmaz", udata->nmaxevent,udata->nz);
        }
        if(udata->nx>0) {
            initField2(&x, "x", udata->nt,udata->nx);
            initField2(&xdot, "xdot", 1,udata->nx);
            initField2(&J, "J", udata->nx,udata->nx);
        }
        if(udata->ny>0) {
            initField2(&y, "y", udata->nt,udata->ny);
            initField2(&sigmay, "sigmay", udata->nt,udata->ny);
            if (udata->sensi_meth == AMICI_SENSI_SS) {
                initField2(&dydp, "dydp", udata->ny,udata->nplist);
                initField2(&dydx, "dydx", udata->ny,udata->nx);
                initField2(&dxdotdp, "dxdotdp", udata->nx,udata->nplist);
            }
        }
        if(udata->sensi >= AMICI_SENSI_ORDER_FIRST) {
            initField2(&sllh, "sllh", udata->nplist,1);

            if (udata->sensi_meth == AMICI_SENSI_FSA) {
                initField3(&sx, "sx", udata->nt,udata->nx,udata->nplist);
                if(udata->ny>0) {
                    initField3(&sy, "sy", udata->nt,udata->ny,udata->nplist);
                    initField3(&ssigmay, "ssigmay", udata->nt,udata->ny,udata->nplist);
                }
                if((udata->nz>0) & (udata->ne>0)){
                    initField3(&srz, "srz", udata->nmaxevent,udata->nz,udata->nplist);
                    if(udata->sensi >= AMICI_SENSI_ORDER_SECOND){
                        initField4(&s2rz, "s2rz", udata->nmaxevent,udata->nz,udata->nplist,udata->nplist);
                    }
                    initField3(&sz, "sz", udata->nmaxevent,udata->nz,udata->nplist);
                    initField3(&ssigmaz, "ssigmaz", udata->nmaxevent,udata->nz,udata->nplist);
                }
            }

            if (udata->sensi_meth == AMICI_SENSI_ASA) {
                if(udata->ny>0) {
                    initField3(&ssigmay, "ssigmay", udata->nt,udata->ny,udata->nplist);
                }
                if((udata->nz>0) & (udata->ne>0)){
                    initField3(&ssigmaz, "ssigmaz", udata->nmaxevent,udata->nz,udata->nplist);
                }
                initField2(&numstepsB, "numstepsB", udata->nt, 1);
                initField2(&numrhsevalsB, "numrhsevalsB", udata->nt, 1);
                initField2(&numerrtestfailsB, "numerrtestfailsB",udata->nt, 1);
                initField2(&numnonlinsolvconvfailsB, "numnonlinsolvconvfailsB", udata->nt, 1);
            }

            if(udata->sensi >= AMICI_SENSI_ORDER_SECOND) {
                initField2(&s2llh, "s2llh", udata->ng-1,udata->nplist);
            }
        }
    }
}

void ReturnData::initField1(double **fieldPointer, const char *fieldName, int dim)
{
    *fieldPointer = new double[dim]();
}

void ReturnData::initField2(double **fieldPointer, const char *fieldName, int dim1, int dim2)
{
    *fieldPointer = new double[(dim1) * (dim2)]();

}

void ReturnData::initField3(double **fieldPointer, const char *fieldName, int dim1, int dim2, int dim3)
{
    *fieldPointer = new double[(dim1) * (dim2) * (dim3)]();
}

void ReturnData::initField4(double **fieldPointer, const char *fieldName, int dim1, int dim2, int dim3, int dim4)
{
    *fieldPointer = new double[(dim1) * (dim2) * (dim3) * (dim4)]();
}
<|endoftext|>


<commit_before>#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <asio.hpp>

#include <cstring>
#include <cassert>

#include "realm.hxx"

using namespace std;

static const asio::ip::address_v4 nativeIpv4Address(
  asio::ip::address_v4::from_string("00.01.02.03"));
static const asio::ip::address_v6 nativeIpv6Address(
  asio::ip::address_v6::from_string(
    "0100:0302:0504:0706:0908:0B0A:0D0C:0F0E"));

template<unsigned Len, typename T>
const byte* getAddressEncoding(const T& t) {
  static byte data[Len];
  const byte* raw = reinterpret_cast<const byte*>(&t[0]);
  bool isNativeCorrect = true;
  for (unsigned i = 0; i < Len; ++i) {
    data[raw[i]] = i;
    isNativeCorrect &= (raw[i] == i);
  }

  return isNativeCorrect? NULL : data;
}

//This is not reentrant, but this won't be an issue since this program is not
//multi-threaded.
template<typename T,
         T (asio::ip::address::* F)() const>
const byte* implGetBytesFromAddress(const asio::ip::address& addr) {
  static typename T::bytes_type t;
  t = (addr.*F)().to_bytes();
  return reinterpret_cast<const byte*>(&t[0]);
}

const Realm Realm::v4(4,
                      getAddressEncoding<4>(
                        nativeIpv4Address.to_bytes()),
                      implGetBytesFromAddress<asio::ip::address_v4,
                                              &asio::ip::address::to_v4>);
const Realm Realm::v6(16,
                      getAddressEncoding<16>(
                        nativeIpv6Address.to_bytes()),
                      implGetBytesFromAddress<asio::ip::address_v6,
                                              &asio::ip::address::to_v6>);

Realm::Realm(unsigned l, const byte* enc,
             const byte* (*gbfa)(const asio::ip::address&))
: getBytesFromAddress(gbfa),
  addressSize(l), addressEncoding(enc)
{ }

void Realm::encodeAddress(byte* dst, const asio::ip::address& addr) const {
  const byte* src = getBytesFromAddress(addr);
  if (addressEncoding)
    for (unsigned i = 0; i < addressSize; ++i)
      dst[addressEncoding[i]] = src[i];
  else
    memcpy(dst, src, addressSize);
}

void Realm::decodeAddress(asio::ip::address& addr, const byte* src) const {
  byte* dst = const_cast<byte*>(getBytesFromAddress(addr));
  if (addressEncoding)
    for (unsigned i = 0; i < addressSize; ++i)
      dst[i] = src[addressEncoding[i]];
  else
    memcpy(dst, src, addressSize);
}

<commit_msg>Fix use of IP addresses which make asio throw.<commit_after>#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <asio.hpp>

#include <cstring>
#include <cassert>

#include "realm.hxx"

using namespace std;

static const asio::ip::address_v4 nativeIpv4Address(
  asio::ip::address_v4::from_string("1.2.3.4"));
static const asio::ip::address_v6 nativeIpv6Address(
  asio::ip::address_v6::from_string(
    "0201:0403:0605:0807:0A09:0C0B:0E0D:100F"));

template<unsigned Len, typename T>
const byte* getAddressEncoding(const T& t) {
  static byte data[Len];
  const byte* raw = reinterpret_cast<const byte*>(&t[0]);
  bool isNativeCorrect = true;
  for (unsigned i = 0; i < Len; ++i) {
    data[raw[i]-1] = i;
    isNativeCorrect &= (raw[i] == i+1);
  }

  return isNativeCorrect? NULL : data;
}

//This is not reentrant, but this won't be an issue since this program is not
//multi-threaded.
template<typename T,
         T (asio::ip::address::* F)() const>
const byte* implGetBytesFromAddress(const asio::ip::address& addr) {
  static typename T::bytes_type t;
  t = (addr.*F)().to_bytes();
  return reinterpret_cast<const byte*>(&t[0]);
}

const Realm Realm::v4(4,
                      getAddressEncoding<4>(
                        nativeIpv4Address.to_bytes()),
                      implGetBytesFromAddress<asio::ip::address_v4,
                                              &asio::ip::address::to_v4>);
const Realm Realm::v6(16,
                      getAddressEncoding<16>(
                        nativeIpv6Address.to_bytes()),
                      implGetBytesFromAddress<asio::ip::address_v6,
                                              &asio::ip::address::to_v6>);

Realm::Realm(unsigned l, const byte* enc,
             const byte* (*gbfa)(const asio::ip::address&))
: getBytesFromAddress(gbfa),
  addressSize(l), addressEncoding(enc)
{ }

void Realm::encodeAddress(byte* dst, const asio::ip::address& addr) const {
  const byte* src = getBytesFromAddress(addr);
  if (addressEncoding)
    for (unsigned i = 0; i < addressSize; ++i)
      dst[addressEncoding[i]] = src[i];
  else
    memcpy(dst, src, addressSize);
}

void Realm::decodeAddress(asio::ip::address& addr, const byte* src) const {
  byte* dst = const_cast<byte*>(getBytesFromAddress(addr));
  if (addressEncoding)
    for (unsigned i = 0; i < addressSize; ++i)
      dst[i] = src[addressEncoding[i]];
  else
    memcpy(dst, src, addressSize);
}

<|endoftext|>