<commit_before>// Copyright 2014 eric schkufza
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <cmath>

#include <algorithm>
#include <array>
#include <limits>

#include "src/cost/cost_function.h"

using namespace cpputil;
using namespace std;
using namespace x64asm;

namespace {

array<stoke::Cost, 3976> latencies_ {{
#include "src/cost/tables/haswell_latency.h"
  }
};

} // namespace

namespace stoke {

CostFunction& CostFunction::set_target(const Cfg& target, bool stack_out, bool heap_out) {
  assert(sandbox_ != nullptr);

  live_out_ = target.live_outs();
  stack_out_ = stack_out;
  heap_out_ = heap_out;

  reference_out_.clear();
  recompute_defs(target.live_outs(), target_gp_out_, target_sse_out_);

  sandbox_->run(target);
  for (auto i = sandbox_->result_begin(), ie = sandbox_->result_end(); i != ie; ++i) {
    reference_out_.push_back(*i);
  }
  return *this;
}

void CostFunction::recompute_defs(const RegSet& rs, vector<R64>& gps, vector<Xmm>& sses) {
  gps.clear();
	for (const auto& r : rls) {
		if (rs.contains(r)) {
			gps.push_back(r64s[r]);
		}
	}
	for (const auto& r : rbs) {
		if (rs.contains(r)) {
			gps.push_back(r64s[r]);
		}
	}

  sses.clear();
  for (const auto& s : xmms) {
    if (rs.contains(s)) {
      sses.push_back(s);
    }
	}
}

Cost CostFunction::evaluate_correctness(const Cfg& cfg, Cost max) {
  switch (reduction_) {
    case Reduction::MAX:
      return max_correctness(cfg, max);
    case Reduction::SUM:
      return sum_correctness(cfg, max);
		case Reduction::EXTENSION:
			return extension_correctness(cfg, max);
    default:
      assert(false);
      return 0;
  }
}

Cost CostFunction::max_correctness(const Cfg& cfg, Cost max) {
  Cost res = 0;

  sandbox_->compile(cfg);
	recompute_defs(cfg.def_outs(), rewrite_gp_out_, rewrite_sse_out_);

	size_t i = 0;
  for (size_t ie = sandbox_->size(); res < max && i < ie; ++i) {
    sandbox_->run_one(i);
    res = std::max(res, evaluate_error(reference_out_[i], *(sandbox_->get_result(i))));
    assert(res <= max_testcase_cost);
  }
	testcases_evaluated_ = i;

	assert(res <= max_correctness_cost);
  return res;
}

Cost CostFunction::sum_correctness(const Cfg& cfg, Cost max) {
  Cost res = 0;

  sandbox_->compile(cfg);
	recompute_defs(cfg.def_outs(), rewrite_gp_out_, rewrite_sse_out_);

	size_t i = 0;
  for (size_t ie = sandbox_->size(); res < max && i < ie; ++i) {
    sandbox_->run_one(i);
    const auto err = evaluate_error(reference_out_[i], *(sandbox_->get_result(i)));
    assert(err <= max_testcase_cost);

    res += err;
  }
	testcases_evaluated_ = i;

	assert(res <= max_correctness_cost);
  return res;
}

Cost CostFunction::extension_correctness(const Cfg& cfg, Cost max) {
	Cost res = 0;

	// Add user-defined implementation here ...

	// This method is not required to examine all testcases. Implementations
	// that compute res iteratively may stop executing and return max once res
	// equals or exceeds that value.

	// This method should set the testcases_evaluated_ attribute to reflect the
	// number of testcases that were examined prior to a possible early exit. The
	// final testcase that was used (testcases_evaluted_-1) is returned by a hold_out
	// verifier as a counter-example following a failed proof of correctness (a hold-out
	// verifier uses a max cost of 0; for non-zero results, the evaluation of this testcase
	// is presumably responsible for res exceeding max).
	testcases_evaluated_ = 1;

	// Invariant 1: 0 < testcases_evaluated_ < sandbox_->size()
	assert(testcases_evaluated_ > 0);
	assert(testcases_evaluated_ < sandbox_->size());

	// Invariant 2: Return value should not exceed max_correctness_cost
	assert(res <= max_correctness_cost);

	return res;
}

Cost CostFunction::evaluate_error(const CpuState& t, const CpuState& r) const {
  if (t.code != r.code) {
    return sig_penalty_;
  }

  Cost cost = 0;
  cost += gp_error(t.gp, r.gp);
  cost += sse_error(t.sse, r.sse);
  if (stack_out_) {
    cost += mem_error(t.stack, r.stack);
  }
  if (heap_out_) {
    cost += mem_error(t.heap, r.heap);
  }

  return cost;
}

Cost CostFunction::gp_error(const Regs& t, const Regs& r) const {
  Cost cost = 0;

  for (const auto& r_t : target_gp_out_) {
    auto delta = max_error_cost;
    const auto val_t = t[r_t].get_fixed_quad(0);

    for (const auto& r_r : rewrite_gp_out_) {
      if (r_t != r_r && !relax_reg_) {
        continue;
      }

      const auto val_r = r[r_r].get_fixed_quad(0);
      const auto eval = evaluate_distance(val_t, val_r) + ((r_t == r_r) ? 0 : misalign_penalty_);

      delta = min(delta, eval);
    }
    cost += delta;
  }

  return cost;
}

Cost CostFunction::sse_error(const Regs& t, const Regs& r) const {
  Cost cost = 0;

  for (size_t i = 0; i < sse_count_; ++i) {
    for (const auto& s_t : target_sse_out_) {
      auto delta = max_error_cost;
      uint64_t val_t = 0;
      switch (sse_width_) {
        case 1: val_t = t[s_t].get_fixed_byte(i);
          break;
        case 2: val_t = t[s_t].get_fixed_word(i);
          break;
        case 4: val_t = t[s_t].get_fixed_double(i);
          break;
        case 8: val_t = t[s_t].get_fixed_quad(i);
          break;
        default: assert(false);
          break;
      }

      for (const auto& s_r : rewrite_sse_out_) {
        if (s_t != s_r && !relax_reg_) {
          continue;
        }

        uint64_t val_r = 0;
        switch (sse_width_) {
          case 1: val_r = r[s_r].get_fixed_byte(i);
            break;
          case 2: val_r = r[s_r].get_fixed_word(i);
            break;
          case 4: val_r = r[s_r].get_fixed_double(i);
            break;
          case 8: val_r = r[s_r].get_fixed_quad(i);
            break;
          default: assert(false);
            break;
        }

        const auto eval = evaluate_distance(val_t, val_r) + ((s_t == s_r) ? 0 : misalign_penalty_);
        delta = min(delta, eval);
      }
      cost += delta;
    }
  }

  return cost;
}

Cost CostFunction::mem_error(const Memory& t, const Memory& r) const {
  Cost cost = 0;

  for (auto i = t.defined_begin(), ie = t.defined_end(); i != ie; ++i) {
    if (relax_mem_) {
      Cost delta = max_error_cost;
      for (auto j = t.defined_begin(), je = t.defined_end(); j != je; ++j) {
        delta = min(delta, evaluate_distance(t[*i], r[*j]));
      }
      cost += delta;
    } else {
      cost += evaluate_distance(t[*i], r[*i]);
    }
  }

  return cost;
}

Cost CostFunction::evaluate_distance(uint64_t x, uint64_t y) const {
	switch(distance_) {
		case Distance::HAMMING:
			return hamming_distance(x, y);
		case Distance::ULP:
			return ulp_distance(x, y);
		case Distance::EXTENSION:
			return extension_distance(x, y);
		default:
			assert(false);
			return 0;
	}
}

Cost CostFunction::ulp_distance(uint64_t x, uint64_t y) const {
	auto t = *((int64_t*)&x);
	t = t < 0 ? numeric_limits<int64_t>::min() - t : t;

	auto r = *((int64_t*)&y);
	r = r < 0 ? numeric_limits<int64_t>::min() - r : r;

	uint64_t ulp = t >= r ? t - r : r - t;
	ulp = ulp < min_ulp_ ? 0 : ulp - min_ulp_;

	return ulp > max_error_cost ? max_error_cost : ulp;
}

Cost CostFunction::extension_distance(uint64_t x, uint64_t y) const {
	Cost res = 0;

	// Add user-defined implementation here ...

	// Invariant 1: Return value should not exceed max_error_cost

	return res;
}

Cost CostFunction::evaluate_performance(const Cfg& cfg, Cost max) const {
  switch (pterm_) {
    case PerformanceTerm::SIZE:
      return size_performance(cfg);
    case PerformanceTerm::LATENCY:
      return latency_performance(cfg);
		case PerformanceTerm::EXTENSION:
			return extension_performance(cfg);
    default:
      assert(false);
      return 0;
  }
}

Cost CostFunction::size_performance(const Cfg& cfg) const {
  Cost size = 0;

  const auto& code = cfg.get_code();
  for (auto b = ++cfg.reachable_begin(), be = cfg.reachable_end(); b != be; ++b) {
		if (cfg.is_exit(*b)) {
			continue;
		}

    const auto first = cfg.get_index(Cfg::loc_type(*b, 0));
    for (size_t i = first, ie = first + cfg.num_instrs(*b); i < ie; ++i) {
      if (!code[i].is_nop()) {
        size++;
      }
    }
  }

  return size;
}

Cost CostFunction::latency_performance(const Cfg& cfg) const {
  Cost latency = 0;

  const auto& code = cfg.get_code();
  for (auto b = ++cfg.reachable_begin(), be = cfg.reachable_end(); b != be; ++b) {
		if (cfg.is_exit(*b)) {
			continue;
		}

    Cost block_latency = 0;
    const auto first = cfg.get_index(Cfg::loc_type(*b, 0));
    for (size_t i = first, ie = first + cfg.num_instrs(*b); i < ie; ++i) {
      block_latency += latencies_[code[i].get_opcode()];
    }

    latency += block_latency * pow(nesting_penalty_, cfg.nesting_depth(*b));
  }

  return latency;
}

Cost CostFunction::extension_performance(const Cfg& cfg) const {
	Cost res = 0;

	// Add user-defined implementation here ...

	// Invariant: Return value should not exceed max_performance_cost
	assert(res <= max_performance_cost);

	return res;
}

} // namespace stoke
<commit_msg>Bug fix. See issue 9.<commit_after>// Copyright 2014 eric schkufza
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <cmath>

#include <algorithm>
#include <array>
#include <limits>

#include "src/cost/cost_function.h"

using namespace cpputil;
using namespace std;
using namespace x64asm;

namespace {

array<stoke::Cost, 3976> latencies_ {{
#include "src/cost/tables/haswell_latency.h"
  }
};

} // namespace

namespace stoke {

CostFunction& CostFunction::set_target(const Cfg& target, bool stack_out, bool heap_out) {
  assert(sandbox_ != nullptr);

  live_out_ = target.live_outs();
  stack_out_ = stack_out;
  heap_out_ = heap_out;

  reference_out_.clear();
  recompute_defs(target.live_outs(), target_gp_out_, target_sse_out_);

  sandbox_->run(target);
  for (auto i = sandbox_->result_begin(), ie = sandbox_->result_end(); i != ie; ++i) {
    reference_out_.push_back(*i);
  }
  return *this;
}

void CostFunction::recompute_defs(const RegSet& rs, vector<R64>& gps, vector<Xmm>& sses) {
  gps.clear();
	for (const auto& r : rls) {
		if (rs.contains(r)) {
			gps.push_back(r64s[r]);
		}
	}
	for (const auto& r : rbs) {
		if (rs.contains(r)) {
			gps.push_back(r64s[r]);
		}
	}

  sses.clear();
  for (const auto& s : xmms) {
    if (rs.contains(s)) {
      sses.push_back(s);
    }
	}
}

Cost CostFunction::evaluate_correctness(const Cfg& cfg, Cost max) {
  switch (reduction_) {
    case Reduction::MAX:
      return max_correctness(cfg, max);
    case Reduction::SUM:
      return sum_correctness(cfg, max);
		case Reduction::EXTENSION:
			return extension_correctness(cfg, max);
    default:
      assert(false);
      return 0;
  }
}

Cost CostFunction::max_correctness(const Cfg& cfg, Cost max) {
  Cost res = 0;

  sandbox_->compile(cfg);
	recompute_defs(cfg.def_outs(), rewrite_gp_out_, rewrite_sse_out_);

	size_t i = 0;
  for (size_t ie = sandbox_->size(); res < max && i < ie; ++i) {
    sandbox_->run_one(i);
    res = std::max(res, evaluate_error(reference_out_[i], *(sandbox_->get_result(i))));
    assert(res <= max_testcase_cost);
  }
	testcases_evaluated_ = i;

	assert(res <= max_correctness_cost);
  return res;
}

Cost CostFunction::sum_correctness(const Cfg& cfg, Cost max) {
  Cost res = 0;

  sandbox_->compile(cfg);
	recompute_defs(cfg.def_outs(), rewrite_gp_out_, rewrite_sse_out_);

	size_t i = 0;
  for (size_t ie = sandbox_->size(); res < max && i < ie; ++i) {
    sandbox_->run_one(i);
    const auto err = evaluate_error(reference_out_[i], *(sandbox_->get_result(i)));
    assert(err <= max_testcase_cost);

    res += err;
  }
	testcases_evaluated_ = i;

	assert(res <= max_correctness_cost);
  return res;
}

Cost CostFunction::extension_correctness(const Cfg& cfg, Cost max) {
	Cost res = 0;

	// Add user-defined implementation here ...

	// This method is not required to examine all testcases. Implementations
	// that compute res iteratively may stop executing and return max once res
	// equals or exceeds that value.

	// This method should set the testcases_evaluated_ attribute to reflect the
	// number of testcases that were examined prior to a possible early exit. The
	// final testcase that was used (testcases_evaluted_-1) is returned by a hold_out
	// verifier as a counter-example following a failed proof of correctness (a hold-out
	// verifier uses a max cost of 0; for non-zero results, the evaluation of this testcase
	// is presumably responsible for res exceeding max).
	testcases_evaluated_ = 1;

	// Invariant 1: 0 < testcases_evaluated_ < sandbox_->size()
	assert(testcases_evaluated_ > 0);
	assert(testcases_evaluated_ < sandbox_->size());

	// Invariant 2: Return value should not exceed max_correctness_cost
	assert(res <= max_correctness_cost);

	return res;
}

Cost CostFunction::evaluate_error(const CpuState& t, const CpuState& r) const {
  if (t.code != r.code) {
    return sig_penalty_;
  }

  Cost cost = 0;
  cost += gp_error(t.gp, r.gp);
  cost += sse_error(t.sse, r.sse);
  if (stack_out_) {
    cost += mem_error(t.stack, r.stack);
  }
  if (heap_out_) {
    cost += mem_error(t.heap, r.heap);
  }

  return cost;
}

Cost CostFunction::gp_error(const Regs& t, const Regs& r) const {
  Cost cost = 0;

  for (const auto& r_t : target_gp_out_) {
    auto delta = max_error_cost;
    const auto val_t = t[r_t].get_fixed_quad(0);

    for (const auto& r_r : rewrite_gp_out_) {
      if (r_t != r_r && !relax_reg_) {
        continue;
      }

      const auto val_r = r[r_r].get_fixed_quad(0);
      const auto eval = evaluate_distance(val_t, val_r) + ((r_t == r_r) ? 0 : misalign_penalty_);

      delta = min(delta, eval);
    }
    cost += delta;
  }

  return cost;
}

Cost CostFunction::sse_error(const Regs& t, const Regs& r) const {
  Cost cost = 0;

  for (size_t i = 0; i < sse_count_; ++i) {
    for (const auto& s_t : target_sse_out_) {
      auto delta = max_error_cost;
      uint64_t val_t = 0;
      switch (sse_width_) {
        case 1: val_t = t[s_t].get_fixed_byte(i);
          break;
        case 2: val_t = t[s_t].get_fixed_word(i);
          break;
        case 4: val_t = t[s_t].get_fixed_double(i);
          break;
        case 8: val_t = t[s_t].get_fixed_quad(i);
          break;
        default: assert(false);
          break;
      }

      for (const auto& s_r : rewrite_sse_out_) {
        if (s_t != s_r && !relax_reg_) {
          continue;
        }

        uint64_t val_r = 0;
        switch (sse_width_) {
          case 1: val_r = r[s_r].get_fixed_byte(i);
            break;
          case 2: val_r = r[s_r].get_fixed_word(i);
            break;
          case 4: val_r = r[s_r].get_fixed_double(i);
            break;
          case 8: val_r = r[s_r].get_fixed_quad(i);
            break;
          default: assert(false);
            break;
        }

        const auto eval = evaluate_distance(val_t, val_r) + ((s_t == s_r) ? 0 : misalign_penalty_);
        delta = min(delta, eval);
      }
      cost += delta;
    }
  }

  return cost;
}

Cost CostFunction::mem_error(const Memory& t, const Memory& r) const {
  Cost cost = 0;

  for (auto i = t.defined_begin(), ie = t.defined_end(); i != ie; ++i) {
    if (relax_mem_) {
      Cost delta = max_error_cost;
      for (auto j = t.defined_begin(), je = t.defined_end(); j != je; ++j) {
        delta = min(delta, evaluate_distance(t[*i], r[*j]));
      }
      cost += delta;
    } else {
      cost += evaluate_distance(t[*i], r[*i]);
    }
  }

  return cost;
}

Cost CostFunction::evaluate_distance(uint64_t x, uint64_t y) const {
	switch(distance_) {
		case Distance::HAMMING:
			return hamming_distance(x, y);
		case Distance::ULP:
			return ulp_distance(x, y);
		case Distance::EXTENSION:
			return extension_distance(x, y);
		default:
			assert(false);
			return 0;
	}
}

Cost CostFunction::ulp_distance(uint64_t x, uint64_t y) const {
	auto t = *((int64_t*)&x);
	t = t < 0 ? numeric_limits<int64_t>::min() - t : t;

	auto r = *((int64_t*)&y);
	r = r < 0 ? numeric_limits<int64_t>::min() - r : r;

	uint64_t ulp = t >= r ? t - r : r - t;
	ulp = ulp < min_ulp_ ? 0 : ulp - min_ulp_;

	return ulp > max_error_cost ? max_error_cost : ulp;
}

Cost CostFunction::extension_distance(uint64_t x, uint64_t y) const {
	Cost res = 0;

	// Add user-defined implementation here ...

	// Invariant 1: Return value should not exceed max_error_cost

	return res;
}

Cost CostFunction::evaluate_performance(const Cfg& cfg, Cost max) const {
  switch (pterm_) {
    case PerformanceTerm::SIZE:
      return size_performance(cfg);
    case PerformanceTerm::LATENCY:
      return latency_performance(cfg);
		case PerformanceTerm::EXTENSION:
			return extension_performance(cfg);
    default:
      assert(false);
      return 0;
  }
}

Cost CostFunction::size_performance(const Cfg& cfg) const {
  Cost size = 0;

  const auto& code = cfg.get_code();
  for (auto b = ++cfg.reachable_begin(), be = cfg.reachable_end(); b != be; ++b) {
		if (cfg.is_exit(*b)) {
			continue;
		}

    const auto first = cfg.get_index(Cfg::loc_type(*b, 0));
    for (size_t i = first, ie = first + cfg.num_instrs(*b); i < ie; ++i) {
      if (!code[i].is_nop()) {
        size++;
      }
    }
  }

  return size;
}

Cost CostFunction::latency_performance(const Cfg& cfg) const {
  Cost latency = 0;

  const auto& code = cfg.get_code();
  for (auto b = ++cfg.reachable_begin(), be = cfg.reachable_end(); b != be; ++b) {
		if (cfg.is_exit(*b)) {
			continue;
		}

    Cost block_latency = 0;
    const auto first = cfg.get_index(Cfg::loc_type(*b, 0));
    for (size_t i = first, ie = first + cfg.num_instrs(*b); i < ie; ++i) {
			if (!code[i].is_nop()) {
        block_latency += latencies_[code[i].get_opcode()];
			}
    }

    latency += block_latency * pow(nesting_penalty_, cfg.nesting_depth(*b));
  }

  return latency;
}

Cost CostFunction::extension_performance(const Cfg& cfg) const {
	Cost res = 0;

	// Add user-defined implementation here ...

	// Invariant: Return value should not exceed max_performance_cost
	assert(res <= max_performance_cost);

	return res;
}

} // namespace stoke
<|endoftext|>


<commit_before>// Copyright (c) 2006, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include <cassert>
#include <dlfcn.h>
#include <curl/curl.h>
#include <curl/easy.h>
#include <curl/types.h>

#include "common/linux/http_upload.h"

namespace {

// Callback to get the response data from server.
static size_t WriteCallback(void *ptr, size_t size,
                            size_t nmemb, void *userp) {
  if (!userp)
    return 0;

  std::string *response = reinterpret_cast<std::string *>(userp);
  size_t real_size = size * nmemb;
  response->append(reinterpret_cast<char *>(ptr), real_size);
  return real_size;
}

}  // namespace

namespace google_breakpad {

static const char kUserAgent[] = "Breakpad/1.0 (Linux)";

// static
bool HTTPUpload::SendRequest(const string &url,
                             const map<string, string> &parameters,
                             const string &upload_file,
                             const string &file_part_name,
                             const string &proxy,
                             const string &proxy_user_pwd,
                             string *response_body,
                             string *error_description) {
  if (!CheckParameters(parameters))
    return false;

  void *curl_lib = dlopen("libcurl.so", RTLD_NOW);
  if (!curl_lib) {
    curl_lib = dlopen("libcurl.so.4", RTLD_NOW);
  }
  if (!curl_lib) {
    curl_lib = dlopen("libcurl.so.3", RTLD_NOW);
  }
  if (!curl_lib) {
    return false;
  }

  CURL* (*curl_easy_init)(void);
  *(void**) (&curl_easy_init) = dlsym(curl_lib, "curl_easy_init");
  CURL *curl = (*curl_easy_init)();
  if (error_description != NULL)
    *error_description = "No Error";

  if (!curl) {
    dlclose(curl_lib);
    return false;
  }

  CURLcode err_code = CURLE_OK;
  CURLcode (*curl_easy_setopt)(CURL *, CURLoption, ...);
  *(void**) (&curl_easy_setopt) = dlsym(curl_lib, "curl_easy_setopt");
  (*curl_easy_setopt)(curl, CURLOPT_URL, url.c_str());
  (*curl_easy_setopt)(curl, CURLOPT_USERAGENT, kUserAgent);
  // Set proxy information if necessary.
  if (!proxy.empty())
    (*curl_easy_setopt)(curl, CURLOPT_PROXY, proxy.c_str());
  if (!proxy_user_pwd.empty())
    (*curl_easy_setopt)(curl, CURLOPT_PROXYUSERPWD, proxy_user_pwd.c_str());

  struct curl_httppost *formpost = NULL;
  struct curl_httppost *lastptr = NULL;
  // Add form data.
  CURLFORMcode (*curl_formadd)(struct curl_httppost **, struct curl_httppost **, ...);
  *(void**) (&curl_formadd) = dlsym(curl_lib, "curl_formadd");
  map<string, string>::const_iterator iter = parameters.begin();
  for (; iter != parameters.end(); ++iter)
    (*curl_formadd)(&formpost, &lastptr,
                 CURLFORM_COPYNAME, iter->first.c_str(),
                 CURLFORM_COPYCONTENTS, iter->second.c_str(),
                 CURLFORM_END);

  // Add form file.
  (*curl_formadd)(&formpost, &lastptr,
               CURLFORM_COPYNAME, file_part_name.c_str(),
               CURLFORM_FILE, upload_file.c_str(),
               CURLFORM_END);

  (*curl_easy_setopt)(curl, CURLOPT_HTTPPOST, formpost);

  // Disable 100-continue header.
  struct curl_slist *headerlist = NULL;
  char buf[] = "Expect:";
  struct curl_slist* (*curl_slist_append)(struct curl_slist *, const char *);
  *(void**) (&curl_slist_append) = dlsym(curl_lib, "curl_slist_append");
  headerlist = (*curl_slist_append)(headerlist, buf);
  (*curl_easy_setopt)(curl, CURLOPT_HTTPHEADER, headerlist);

  if (response_body != NULL) {
    (*curl_easy_setopt)(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
    (*curl_easy_setopt)(curl, CURLOPT_WRITEDATA,
                     reinterpret_cast<void *>(response_body));
  }

  CURLcode (*curl_easy_perform)(CURL *);
  *(void**) (&curl_easy_perform) = dlsym(curl_lib, "curl_easy_perform");
  err_code = (*curl_easy_perform)(curl);
#ifndef NDEBUG
  const char* (*curl_easy_strerror)(CURLcode);
  *(void**) (&curl_easy_strerror) = dlsym(curl_lib, "curl_easy_strerror");
  if (err_code != CURLE_OK)
    fprintf(stderr, "Failed to send http request to %s, error: %s\n",
            url.c_str(),
            (*curl_easy_strerror)(err_code));
#endif
  if (error_description != NULL)
    *error_description = (*curl_easy_strerror)(err_code);

  void (*curl_easy_cleanup)(CURL *);
  *(void**) (&curl_easy_cleanup) = dlsym(curl_lib, "curl_easy_cleanup");
  (*curl_easy_cleanup)(curl);
  if (formpost != NULL) {
    void (*curl_formfree)(struct curl_httppost *);
    *(void**) (&curl_formfree) = dlsym(curl_lib, "curl_formfree");
    (*curl_formfree)(formpost);
  }
  if (headerlist != NULL) {
    void (*curl_slist_free_all)(struct curl_slist *);
    *(void**) (&curl_slist_free_all) = dlsym(curl_lib, "curl_slist_free_all");
    (*curl_slist_free_all)(headerlist);
  }
  dlclose(curl_lib);
  return err_code == CURLE_OK;
}

// static
bool HTTPUpload::CheckParameters(const map<string, string> &parameters) {
  for (map<string, string>::const_iterator pos = parameters.begin();
       pos != parameters.end(); ++pos) {
    const string &str = pos->first;
    if (str.size() == 0)
      return false;  // disallow empty parameter names
    for (unsigned int i = 0; i < str.size(); ++i) {
      int c = str[i];
      if (c < 32 || c == '"' || c > 127) {
        return false;
      }
    }
  }
  return true;
}

}  // namespace google_breakpad
<commit_msg>slight fixup from issue 225<commit_after>// Copyright (c) 2006, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include <cassert>
#include <dlfcn.h>
#include <curl/curl.h>
#include <curl/easy.h>
#include <curl/types.h>

#include "common/linux/http_upload.h"

namespace {

// Callback to get the response data from server.
static size_t WriteCallback(void *ptr, size_t size,
                            size_t nmemb, void *userp) {
  if (!userp)
    return 0;

  std::string *response = reinterpret_cast<std::string *>(userp);
  size_t real_size = size * nmemb;
  response->append(reinterpret_cast<char *>(ptr), real_size);
  return real_size;
}

}  // namespace

namespace google_breakpad {

static const char kUserAgent[] = "Breakpad/1.0 (Linux)";

// static
bool HTTPUpload::SendRequest(const string &url,
                             const map<string, string> &parameters,
                             const string &upload_file,
                             const string &file_part_name,
                             const string &proxy,
                             const string &proxy_user_pwd,
                             string *response_body,
                             string *error_description) {
  if (!CheckParameters(parameters))
    return false;

  void *curl_lib = dlopen("libcurl.so", RTLD_NOW);
  if (!curl_lib) {
    curl_lib = dlopen("libcurl.so.4", RTLD_NOW);
  }
  if (!curl_lib) {
    curl_lib = dlopen("libcurl.so.3", RTLD_NOW);
  }
  if (!curl_lib) {
    return false;
  }

  CURL* (*curl_easy_init)(void);
  *(void**) (&curl_easy_init) = dlsym(curl_lib, "curl_easy_init");
  CURL *curl = (*curl_easy_init)();
  if (error_description != NULL)
    *error_description = "No Error";

  if (!curl) {
    dlclose(curl_lib);
    return false;
  }

  CURLcode err_code = CURLE_OK;
  CURLcode (*curl_easy_setopt)(CURL *, CURLoption, ...);
  *(void**) (&curl_easy_setopt) = dlsym(curl_lib, "curl_easy_setopt");
  (*curl_easy_setopt)(curl, CURLOPT_URL, url.c_str());
  (*curl_easy_setopt)(curl, CURLOPT_USERAGENT, kUserAgent);
  // Set proxy information if necessary.
  if (!proxy.empty())
    (*curl_easy_setopt)(curl, CURLOPT_PROXY, proxy.c_str());
  if (!proxy_user_pwd.empty())
    (*curl_easy_setopt)(curl, CURLOPT_PROXYUSERPWD, proxy_user_pwd.c_str());

  struct curl_httppost *formpost = NULL;
  struct curl_httppost *lastptr = NULL;
  // Add form data.
  CURLFORMcode (*curl_formadd)(struct curl_httppost **, struct curl_httppost **, ...);
  *(void**) (&curl_formadd) = dlsym(curl_lib, "curl_formadd");
  map<string, string>::const_iterator iter = parameters.begin();
  for (; iter != parameters.end(); ++iter)
    (*curl_formadd)(&formpost, &lastptr,
                 CURLFORM_COPYNAME, iter->first.c_str(),
                 CURLFORM_COPYCONTENTS, iter->second.c_str(),
                 CURLFORM_END);

  // Add form file.
  (*curl_formadd)(&formpost, &lastptr,
               CURLFORM_COPYNAME, file_part_name.c_str(),
               CURLFORM_FILE, upload_file.c_str(),
               CURLFORM_END);

  (*curl_easy_setopt)(curl, CURLOPT_HTTPPOST, formpost);

  // Disable 100-continue header.
  struct curl_slist *headerlist = NULL;
  char buf[] = "Expect:";
  struct curl_slist* (*curl_slist_append)(struct curl_slist *, const char *);
  *(void**) (&curl_slist_append) = dlsym(curl_lib, "curl_slist_append");
  headerlist = (*curl_slist_append)(headerlist, buf);
  (*curl_easy_setopt)(curl, CURLOPT_HTTPHEADER, headerlist);

  if (response_body != NULL) {
    (*curl_easy_setopt)(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
    (*curl_easy_setopt)(curl, CURLOPT_WRITEDATA,
                     reinterpret_cast<void *>(response_body));
  }

  CURLcode (*curl_easy_perform)(CURL *);
  *(void**) (&curl_easy_perform) = dlsym(curl_lib, "curl_easy_perform");
  err_code = (*curl_easy_perform)(curl);
  const char* (*curl_easy_strerror)(CURLcode);
  *(void**) (&curl_easy_strerror) = dlsym(curl_lib, "curl_easy_strerror");
#ifndef NDEBUG
  if (err_code != CURLE_OK)
    fprintf(stderr, "Failed to send http request to %s, error: %s\n",
            url.c_str(),
            (*curl_easy_strerror)(err_code));
#endif
  if (error_description != NULL)
    *error_description = (*curl_easy_strerror)(err_code);

  void (*curl_easy_cleanup)(CURL *);
  *(void**) (&curl_easy_cleanup) = dlsym(curl_lib, "curl_easy_cleanup");
  (*curl_easy_cleanup)(curl);
  if (formpost != NULL) {
    void (*curl_formfree)(struct curl_httppost *);
    *(void**) (&curl_formfree) = dlsym(curl_lib, "curl_formfree");
    (*curl_formfree)(formpost);
  }
  if (headerlist != NULL) {
    void (*curl_slist_free_all)(struct curl_slist *);
    *(void**) (&curl_slist_free_all) = dlsym(curl_lib, "curl_slist_free_all");
    (*curl_slist_free_all)(headerlist);
  }
  dlclose(curl_lib);
  return err_code == CURLE_OK;
}

// static
bool HTTPUpload::CheckParameters(const map<string, string> &parameters) {
  for (map<string, string>::const_iterator pos = parameters.begin();
       pos != parameters.end(); ++pos) {
    const string &str = pos->first;
    if (str.size() == 0)
      return false;  // disallow empty parameter names
    for (unsigned int i = 0; i < str.size(); ++i) {
      int c = str[i];
      if (c < 32 || c == '"' || c > 127) {
        return false;
      }
    }
  }
  return true;
}

}  // namespace google_breakpad
<|endoftext|>


<commit_before>// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include <vespa/vespalib/testkit/test_kit.h>
#include <vespa/eval/eval/tensor_function.h>
#include <vespa/eval/instruction/mixed_simple_join_function.h>
#include <vespa/eval/eval/test/eval_fixture.h>
#include <vespa/eval/eval/test/gen_spec.h>

#include <vespa/vespalib/util/stringfmt.h>

using namespace vespalib;
using namespace vespalib::eval;
using namespace vespalib::eval::test;
using namespace vespalib::eval::tensor_function;

using vespalib::make_string_short::fmt;

using Primary = MixedSimpleJoinFunction::Primary;
using Overlap = MixedSimpleJoinFunction::Overlap;

namespace vespalib::eval {

std::ostream &operator<<(std::ostream &os, Primary primary)
{
    switch(primary) {
    case Primary::LHS: return os << "LHS";
    case Primary::RHS: return os << "RHS";
    }
    abort();
}

std::ostream &operator<<(std::ostream &os, Overlap overlap)
{
    switch(overlap) {
    case Overlap::FULL: return os << "FULL";
    case Overlap::INNER: return os << "INNER";
    case Overlap::OUTER: return os << "OUTER";
    }
    abort();
}

}

struct FunInfo {
    using LookFor = MixedSimpleJoinFunction;
    Overlap overlap;
    size_t factor;
    std::optional<Primary> primary;
    bool l_mut;
    bool r_mut;
    std::optional<bool> inplace;
    void verify(const EvalFixture &fixture, const LookFor &fun) const {
        EXPECT_TRUE(fun.result_is_mutable());
        EXPECT_EQUAL(fun.overlap(), overlap);
        EXPECT_EQUAL(fun.factor(), factor);
        if (primary.has_value()) {
            EXPECT_EQUAL(fun.primary(), primary.value());
        }
        if (fun.primary_is_mutable()) {
            if (fun.primary() == Primary::LHS) {
                EXPECT_TRUE(l_mut);
            }
            if (fun.primary() == Primary::RHS) {
                EXPECT_TRUE(r_mut);
            }
        }
        if (inplace.has_value()) {
            EXPECT_EQUAL(fun.inplace(), inplace.value());
        }
        if (fun.inplace()) {
            EXPECT_TRUE(fun.primary_is_mutable());
            size_t idx = (fun.primary() == Primary::LHS) ? 0 : 1;
            EXPECT_EQUAL(fixture.result_value().cells().data,
                         fixture.param_value(idx).cells().data);
            EXPECT_NOT_EQUAL(fixture.result_value().cells().data,
                             fixture.param_value(1-idx).cells().data);
        } else {
            EXPECT_NOT_EQUAL(fixture.result_value().cells().data,
                             fixture.param_value(0).cells().data);
            EXPECT_NOT_EQUAL(fixture.result_value().cells().data,
                             fixture.param_value(1).cells().data);
        }
    }
};

void verify_simple(const vespalib::string &expr, Primary primary, Overlap overlap, size_t factor,
                   bool l_mut, bool r_mut, bool inplace)
{
    TEST_STATE(expr.c_str());
    CellTypeSpace just_double({CellType::DOUBLE}, 2);
    FunInfo details{overlap, factor, primary, l_mut, r_mut, inplace};
    EvalFixture::verify<FunInfo>(expr, {details}, just_double);
}

void verify_optimized(const vespalib::string &expr, Primary primary, Overlap overlap, size_t factor,
                      bool l_mut = false, bool r_mut = false)
{
    TEST_STATE(expr.c_str());
    CellTypeSpace all_types(CellTypeUtils::list_types(), 2);
    FunInfo details{overlap, factor, primary, l_mut, r_mut, std::nullopt};
    EvalFixture::verify<FunInfo>(expr, {details}, all_types);
}

void verify_optimized(const vespalib::string &expr, Overlap overlap, size_t factor,
                      bool l_mut = false, bool r_mut = false)
{
    TEST_STATE(expr.c_str());
    CellTypeSpace all_types(CellTypeUtils::list_types(), 2);
    FunInfo details{overlap, factor, std::nullopt, l_mut, r_mut, std::nullopt};
    EvalFixture::verify<FunInfo>(expr, {details}, all_types);
}

void verify_not_optimized(const vespalib::string &expr) {
    TEST_STATE(expr.c_str());
    CellTypeSpace just_double({CellType::DOUBLE}, 2);
    EvalFixture::verify<FunInfo>(expr, {}, just_double);
}

TEST("require that basic join is optimized") {
    TEST_DO(verify_optimized("y5+y5$2", Primary::RHS, Overlap::FULL, 1));
}

TEST("require that inplace is preferred") {
    TEST_DO(verify_simple("y5+y5$2", Primary::RHS, Overlap::FULL, 1, false, false, false));
    TEST_DO(verify_simple("y5+@y5$2", Primary::RHS, Overlap::FULL, 1, false, true, true));
    TEST_DO(verify_simple("@y5+@y5$2", Primary::RHS, Overlap::FULL, 1, true, true, true));
    TEST_DO(verify_simple("@y5+y5$2", Primary::LHS, Overlap::FULL, 1, true, false, true));
}

TEST("require that unit join is optimized") {
    TEST_DO(verify_optimized("a1b1c1+x1y1z1", Primary::RHS, Overlap::FULL, 1));
}

TEST("require that trivial dimensions do not affect overlap calculation") {
    TEST_DO(verify_optimized("c5d1+b1c5", Primary::RHS, Overlap::FULL, 1));
}

TEST("require that outer nesting is preferred to inner nesting") {
    TEST_DO(verify_optimized("a1b1c1+y5", Primary::RHS, Overlap::OUTER, 5));
}

TEST("require that non-subset join is not optimized") {
    TEST_DO(verify_not_optimized("y5+z3"));
}

TEST("require that subset join with complex overlap is not optimized") {
    TEST_DO(verify_not_optimized("x3y5z3+y5"));
}

struct LhsRhs {
    vespalib::string lhs;
    vespalib::string rhs;
    size_t lhs_size;
    size_t rhs_size;
    Overlap overlap;
    size_t factor;
    LhsRhs(const vespalib::string &lhs_in, const vespalib::string &rhs_in,
           size_t lhs_size_in, size_t rhs_size_in, Overlap overlap_in) noexcept
        : lhs(lhs_in), rhs(rhs_in), lhs_size(lhs_size_in), rhs_size(rhs_size_in), overlap(overlap_in), factor(1)
    {
        if (lhs_size > rhs_size) {
            ASSERT_EQUAL(lhs_size % rhs_size, 0u);
            factor = (lhs_size / rhs_size);
        } else {
            ASSERT_EQUAL(rhs_size % lhs_size, 0u);
            factor = (rhs_size / lhs_size);
        }
    }
};

vespalib::string adjust_param(const vespalib::string &str, bool mut_cells, bool is_rhs) {
    vespalib::string result = str;
    if (mut_cells) {
        result = "@" + result;
    }
    if (is_rhs) {
        result += "$2";
    }
    return result;
}

TEST("require that various parameter combinations work") {
    for (bool left_mut: {false, true}) {
        for (bool right_mut: {false, true}) {
            for (const char *op_pattern: {"%s+%s", "%s-%s", "%s*%s"}) {
                for (const LhsRhs &params:
                         {       LhsRhs("y5",   "y5", 5,  5, Overlap::FULL),
                                 LhsRhs("y5", "x3y5", 5, 15, Overlap::INNER),
                                 LhsRhs("y5", "y5z3", 5, 15, Overlap::OUTER),
                                 LhsRhs("x3y5", "y5", 15, 5, Overlap::INNER),
                                 LhsRhs("y5z3", "y5", 15, 5, Overlap::OUTER)})
                {
                    vespalib::string left = adjust_param(params.lhs, left_mut, false);
                    vespalib::string right = adjust_param(params.rhs, right_mut, true);
                    vespalib::string expr = fmt(op_pattern, left.c_str(), right.c_str());
                    TEST_STATE(expr.c_str());
                    verify_optimized(expr, params.overlap, params.factor,
                                     left_mut, right_mut);
                }
            }
        }
    }
}

TEST("require that scalar values are not optimized") {
    TEST_DO(verify_not_optimized("reduce(v3,sum)+reduce(v4,sum)"));
    TEST_DO(verify_not_optimized("reduce(v3,sum)+y5"));
    TEST_DO(verify_not_optimized("y5+reduce(v3,sum)"));
    TEST_DO(verify_not_optimized("reduce(v3,sum)+x3_1"));
    TEST_DO(verify_not_optimized("x3_1+reduce(v3,sum)"));
    TEST_DO(verify_not_optimized("reduce(v3,sum)+x3_1y5z3"));
    TEST_DO(verify_not_optimized("x3_1y5z3+reduce(v3,sum)"));
}

TEST("require that sparse tensors are mostly not optimized") {
    TEST_DO(verify_not_optimized("x3_1+x3_1$2"));
    TEST_DO(verify_not_optimized("x3_1+y5"));
    TEST_DO(verify_not_optimized("y5+x3_1"));
    TEST_DO(verify_not_optimized("x3_1+x3_1y5z3"));
    TEST_DO(verify_not_optimized("x3_1y5z3+x3_1"));
}

TEST("require that sparse tensor joined with trivial dense tensor is optimized") {
    TEST_DO(verify_optimized("x3_1+a1b1c1", Primary::LHS, Overlap::FULL, 1));
    TEST_DO(verify_optimized("a1b1c1+x3_1", Primary::RHS, Overlap::FULL, 1));
}

TEST("require that primary tensor can be empty") {
    TEST_DO(verify_optimized("x0_1y5z3+y5z3", Primary::LHS, Overlap::FULL, 1));
    TEST_DO(verify_optimized("y5z3+x0_1y5z3", Primary::RHS, Overlap::FULL, 1));
}

TEST("require that mixed tensors can be optimized") {
    TEST_DO(verify_not_optimized("x3_1y5z3+x3_1y5z3$2"));
    TEST_DO(verify_optimized("x3_1y5z3+y5z3", Primary::LHS, Overlap::FULL,   1));
    TEST_DO(verify_optimized("x3_1y5z3+y5",   Primary::LHS, Overlap::OUTER,  3));
    TEST_DO(verify_optimized("x3_1y5z3+z3",   Primary::LHS, Overlap::INNER,  5));
    TEST_DO(verify_optimized("y5z3+x3_1y5z3", Primary::RHS, Overlap::FULL,   1));
    TEST_DO(verify_optimized("y5+x3_1y5z3",   Primary::RHS, Overlap::OUTER,  3));
    TEST_DO(verify_optimized("z3+x3_1y5z3",   Primary::RHS, Overlap::INNER,  5));
}

TEST("require that mixed tensors can be inplace") {
    TEST_DO(verify_optimized("@x3_1y5z3+y5z3", Primary::LHS, Overlap::FULL,   1, true, false));
    TEST_DO(verify_optimized("@x3_1y5z3+y5",   Primary::LHS, Overlap::OUTER,  3, true, false));
    TEST_DO(verify_optimized("@x3_1y5z3+z3",   Primary::LHS, Overlap::INNER,  5, true, false));
    TEST_DO(verify_optimized("y5z3+@x3_1y5z3", Primary::RHS, Overlap::FULL,   1, false, true));
    TEST_DO(verify_optimized("y5+@x3_1y5z3",   Primary::RHS, Overlap::OUTER,  3, false, true));
    TEST_DO(verify_optimized("z3+@x3_1y5z3",   Primary::RHS, Overlap::INNER,  5, false, true));
}

TEST_MAIN() { TEST_RUN_ALL(); }
<commit_msg>use our own cell type loops again<commit_after>// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include <vespa/vespalib/testkit/test_kit.h>
#include <vespa/eval/eval/fast_value.h>
#include <vespa/eval/eval/simple_value.h>
#include <vespa/eval/eval/tensor_function.h>
#include <vespa/eval/instruction/mixed_simple_join_function.h>
#include <vespa/eval/eval/test/eval_fixture.h>
#include <vespa/eval/eval/test/gen_spec.h>

#include <vespa/vespalib/util/stringfmt.h>

using namespace vespalib;
using namespace vespalib::eval;
using namespace vespalib::eval::test;
using namespace vespalib::eval::tensor_function;

using vespalib::make_string_short::fmt;

const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get();
const ValueBuilderFactory &test_factory  = SimpleValueBuilderFactory::get();

using Primary = MixedSimpleJoinFunction::Primary;
using Overlap = MixedSimpleJoinFunction::Overlap;

namespace vespalib::eval {

std::ostream &operator<<(std::ostream &os, Primary primary)
{
    switch(primary) {
    case Primary::LHS: return os << "LHS";
    case Primary::RHS: return os << "RHS";
    }
    abort();
}

std::ostream &operator<<(std::ostream &os, Overlap overlap)
{
    switch(overlap) {
    case Overlap::FULL: return os << "FULL";
    case Overlap::INNER: return os << "INNER";
    case Overlap::OUTER: return os << "OUTER";
    }
    abort();
}

}

struct FunInfo {
    using LookFor = MixedSimpleJoinFunction;
    Overlap overlap;
    size_t factor;
    std::optional<Primary> primary;
    bool l_mut;
    bool r_mut;
    std::optional<bool> inplace;
    void verify(const EvalFixture &fixture, const LookFor &fun) const {
        EXPECT_TRUE(fun.result_is_mutable());
        EXPECT_EQUAL(fun.overlap(), overlap);
        EXPECT_EQUAL(fun.factor(), factor);
        if (primary.has_value()) {
            EXPECT_EQUAL(fun.primary(), primary.value());
        }
        if (fun.primary_is_mutable()) {
            if (fun.primary() == Primary::LHS) {
                EXPECT_TRUE(l_mut);
            }
            if (fun.primary() == Primary::RHS) {
                EXPECT_TRUE(r_mut);
            }
        }
        if (inplace.has_value()) {
            EXPECT_EQUAL(fun.inplace(), inplace.value());
        }
        if (fun.inplace()) {
            EXPECT_TRUE(fun.primary_is_mutable());
            size_t idx = (fun.primary() == Primary::LHS) ? 0 : 1;
            EXPECT_EQUAL(fixture.result_value().cells().data,
                         fixture.param_value(idx).cells().data);
            EXPECT_NOT_EQUAL(fixture.result_value().cells().data,
                             fixture.param_value(1-idx).cells().data);
        } else {
            EXPECT_NOT_EQUAL(fixture.result_value().cells().data,
                             fixture.param_value(0).cells().data);
            EXPECT_NOT_EQUAL(fixture.result_value().cells().data,
                             fixture.param_value(1).cells().data);
        }
    }
};

void verify_simple(const vespalib::string &expr, Primary primary, Overlap overlap, size_t factor,
                   bool l_mut, bool r_mut, bool inplace)
{
    TEST_STATE(expr.c_str());
    CellTypeSpace just_double({CellType::DOUBLE}, 2);
    FunInfo details{overlap, factor, primary, l_mut, r_mut, inplace};
    EvalFixture::verify<FunInfo>(expr, {details}, just_double);
}

void verify_optimized(const vespalib::string &expr, Primary primary, Overlap overlap, size_t factor,
                      bool l_mut = false, bool r_mut = false)
{
    TEST_STATE(expr.c_str());
    CellTypeSpace all_types(CellTypeUtils::list_types(), 2);
    FunInfo details{overlap, factor, primary, l_mut, r_mut, std::nullopt};
    EvalFixture::verify<FunInfo>(expr, {details}, all_types);
}

void verify_optimized(const vespalib::string &expr, Overlap overlap, size_t factor,
                      bool l_mut = false, bool r_mut = false)
{
    TEST_STATE(expr.c_str());
    CellTypeSpace all_types(CellTypeUtils::list_types(), 2);
    FunInfo details{overlap, factor, std::nullopt, l_mut, r_mut, std::nullopt};
    EvalFixture::verify<FunInfo>(expr, {details}, all_types);
}

void verify_not_optimized(const vespalib::string &expr) {
    TEST_STATE(expr.c_str());
    CellTypeSpace just_double({CellType::DOUBLE}, 2);
    EvalFixture::verify<FunInfo>(expr, {}, just_double);
}

TEST("require that basic join is optimized") {
    TEST_DO(verify_optimized("y5+y5$2", Primary::RHS, Overlap::FULL, 1));
}

TEST("require that inplace is preferred") {
    TEST_DO(verify_simple("y5+y5$2", Primary::RHS, Overlap::FULL, 1, false, false, false));
    TEST_DO(verify_simple("y5+@y5$2", Primary::RHS, Overlap::FULL, 1, false, true, true));
    TEST_DO(verify_simple("@y5+@y5$2", Primary::RHS, Overlap::FULL, 1, true, true, true));
    TEST_DO(verify_simple("@y5+y5$2", Primary::LHS, Overlap::FULL, 1, true, false, true));
}

TEST("require that unit join is optimized") {
    TEST_DO(verify_optimized("a1b1c1+x1y1z1", Primary::RHS, Overlap::FULL, 1));
}

TEST("require that trivial dimensions do not affect overlap calculation") {
    TEST_DO(verify_optimized("c5d1+b1c5", Primary::RHS, Overlap::FULL, 1));
}

TEST("require that outer nesting is preferred to inner nesting") {
    TEST_DO(verify_optimized("a1b1c1+y5", Primary::RHS, Overlap::OUTER, 5));
}

TEST("require that non-subset join is not optimized") {
    TEST_DO(verify_not_optimized("y5+z3"));
}

TEST("require that subset join with complex overlap is not optimized") {
    TEST_DO(verify_not_optimized("x3y5z3+y5"));
}

struct LhsRhs {
    vespalib::string lhs;
    vespalib::string rhs;
    size_t lhs_size;
    size_t rhs_size;
    Overlap overlap;
    size_t factor;
    LhsRhs(const vespalib::string &lhs_in, const vespalib::string &rhs_in,
           size_t lhs_size_in, size_t rhs_size_in, Overlap overlap_in) noexcept
        : lhs(lhs_in), rhs(rhs_in), lhs_size(lhs_size_in), rhs_size(rhs_size_in), overlap(overlap_in), factor(1)
    {
        if (lhs_size > rhs_size) {
            ASSERT_EQUAL(lhs_size % rhs_size, 0u);
            factor = (lhs_size / rhs_size);
        } else {
            ASSERT_EQUAL(rhs_size % lhs_size, 0u);
            factor = (rhs_size / lhs_size);
        }
    }
};

vespalib::string adjust_param(const vespalib::string &str, bool mut_cells, bool is_rhs) {
    vespalib::string result = str;
    if (mut_cells) {
        result = "@" + result;
    }
    if (is_rhs) {
        result += "$2";
    }
    return result;
}

CellType join_ct(CellType lct, CellType rct) {
    if (lct == CellType::DOUBLE || rct == CellType::DOUBLE) {
        return CellType::DOUBLE;
    } else {
        return CellType::FLOAT;
    }
}

TEST("require that various parameter combinations work") {
    for (CellType lct : CellTypeUtils::list_types()) {
        for (CellType rct : CellTypeUtils::list_types()) {
            for (bool left_mut: {false, true}) {
                for (bool right_mut: {false, true}) {
                    for (const char * expr: {"a+b", "a-b", "a*b"}) {
                        for (const LhsRhs &params:
                                 {       LhsRhs("y5",   "y5", 5,  5, Overlap::FULL),
                                         LhsRhs("y5", "x3y5", 5, 15, Overlap::INNER),
                                         LhsRhs("y5", "y5z3", 5, 15, Overlap::OUTER),
                                         LhsRhs("x3y5", "y5", 15, 5, Overlap::INNER),
                                         LhsRhs("y5z3", "y5", 15, 5, Overlap::OUTER)})
                        {
                            EvalFixture::ParamRepo param_repo;
                            auto a_spec = GenSpec::from_desc(params.lhs).cells(lct).seq(AX_B(0.25, 1.125));
                            auto b_spec = GenSpec::from_desc(params.rhs).cells(rct).seq(AX_B(-0.25, 25.0));
                            if (left_mut) {
                                param_repo.add_mutable("a", a_spec);
                            } else {
                                param_repo.add("a", a_spec);
                            }
                            if (right_mut) {
                                param_repo.add_mutable("b", b_spec);
                            } else {
                                param_repo.add("b", b_spec);
                            }
                            TEST_STATE(expr);
                            CellType result_ct = join_ct(lct, rct);
                            Primary primary = Primary::RHS;
                            if (params.overlap == Overlap::FULL) {
                                bool w_lhs = (lct == result_ct) && left_mut;
                                bool w_rhs = (rct == result_ct) && right_mut;
                                if (w_lhs && !w_rhs) {
                                    primary = Primary::LHS;
                                }
                            } else if (params.lhs_size > params.rhs_size) {
                                primary = Primary::LHS;
                            }
                            bool pri_mut = (primary == Primary::LHS) ? left_mut : right_mut;
                            bool pri_same_ct = (primary == Primary::LHS) ? (lct == result_ct) : (rct == result_ct);
                            bool inplace = (pri_mut && pri_same_ct);
                            auto expect = EvalFixture::ref(expr, param_repo);
                            EvalFixture slow_fixture(prod_factory, expr, param_repo, false);
                            EvalFixture test_fixture(test_factory, expr, param_repo, true, true);
                            EvalFixture fixture(prod_factory, expr, param_repo, true, true);
                            EXPECT_EQUAL(fixture.result(), expect);
                            EXPECT_EQUAL(slow_fixture.result(), expect);
                            EXPECT_EQUAL(test_fixture.result(), expect);
                            auto info = fixture.find_all<FunInfo::LookFor>();
                            ASSERT_EQUAL(info.size(), 1u);
                            FunInfo details{params.overlap, params.factor, primary, left_mut, right_mut, inplace};
                            details.verify(fixture, *info[0]);
                        }
                    }
                }
            }
        }
    }
}

TEST("require that scalar values are not optimized") {
    TEST_DO(verify_not_optimized("reduce(v3,sum)+reduce(v4,sum)"));
    TEST_DO(verify_not_optimized("reduce(v3,sum)+y5"));
    TEST_DO(verify_not_optimized("y5+reduce(v3,sum)"));
    TEST_DO(verify_not_optimized("reduce(v3,sum)+x3_1"));
    TEST_DO(verify_not_optimized("x3_1+reduce(v3,sum)"));
    TEST_DO(verify_not_optimized("reduce(v3,sum)+x3_1y5z3"));
    TEST_DO(verify_not_optimized("x3_1y5z3+reduce(v3,sum)"));
}

TEST("require that sparse tensors are mostly not optimized") {
    TEST_DO(verify_not_optimized("x3_1+x3_1$2"));
    TEST_DO(verify_not_optimized("x3_1+y5"));
    TEST_DO(verify_not_optimized("y5+x3_1"));
    TEST_DO(verify_not_optimized("x3_1+x3_1y5z3"));
    TEST_DO(verify_not_optimized("x3_1y5z3+x3_1"));
}

TEST("require that sparse tensor joined with trivial dense tensor is optimized") {
    TEST_DO(verify_optimized("x3_1+a1b1c1", Primary::LHS, Overlap::FULL, 1));
    TEST_DO(verify_optimized("a1b1c1+x3_1", Primary::RHS, Overlap::FULL, 1));
}

TEST("require that primary tensor can be empty") {
    TEST_DO(verify_optimized("x0_1y5z3+y5z3", Primary::LHS, Overlap::FULL, 1));
    TEST_DO(verify_optimized("y5z3+x0_1y5z3", Primary::RHS, Overlap::FULL, 1));
}

TEST("require that mixed tensors can be optimized") {
    TEST_DO(verify_not_optimized("x3_1y5z3+x3_1y5z3$2"));
    TEST_DO(verify_optimized("x3_1y5z3+y5z3", Primary::LHS, Overlap::FULL,   1));
    TEST_DO(verify_optimized("x3_1y5z3+y5",   Primary::LHS, Overlap::OUTER,  3));
    TEST_DO(verify_optimized("x3_1y5z3+z3",   Primary::LHS, Overlap::INNER,  5));
    TEST_DO(verify_optimized("y5z3+x3_1y5z3", Primary::RHS, Overlap::FULL,   1));
    TEST_DO(verify_optimized("y5+x3_1y5z3",   Primary::RHS, Overlap::OUTER,  3));
    TEST_DO(verify_optimized("z3+x3_1y5z3",   Primary::RHS, Overlap::INNER,  5));
}

TEST("require that mixed tensors can be inplace") {
    TEST_DO(verify_optimized("@x3_1y5z3+y5z3", Primary::LHS, Overlap::FULL,   1, true, false));
    TEST_DO(verify_optimized("@x3_1y5z3+y5",   Primary::LHS, Overlap::OUTER,  3, true, false));
    TEST_DO(verify_optimized("@x3_1y5z3+z3",   Primary::LHS, Overlap::INNER,  5, true, false));
    TEST_DO(verify_optimized("y5z3+@x3_1y5z3", Primary::RHS, Overlap::FULL,   1, false, true));
    TEST_DO(verify_optimized("y5+@x3_1y5z3",   Primary::RHS, Overlap::OUTER,  3, false, true));
    TEST_DO(verify_optimized("z3+@x3_1y5z3",   Primary::RHS, Overlap::INNER,  5, false, true));
}

TEST_MAIN() { TEST_RUN_ALL(); }
<|endoftext|>


<commit_before>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "paddle/framework/lod_tensor.h"
#include "paddle/framework/data_type.h"
#include "paddle/framework/framework.pb.h"

#include "paddle/memory/memcpy.h"
#include "paddle/memory/memory.h"

#include <stdint.h>
#include <string.h>
#include <algorithm>
#include <iterator>

#include <glog/logging.h>

namespace paddle {
namespace framework {

std::ostream &operator<<(std::ostream &os, const LoD &lod) {
  os << "{";
  for (auto &v : lod) {
    os << "{";
    for (auto &i : v) {
      os << i << ",";
    }
    os << "}";
  }
  os << "}";

  return os;
}

LoD SliceLevels(const LoD &in, size_t level_begin, size_t level_end) {
  LoD new_lod;
  new_lod.reserve(level_end - level_begin);
  for (size_t i = level_begin; i < level_end; i++) {
    new_lod.emplace_back(in.at(i));
  }
  // transform the lowest level to absolute offset.
  LoD abs_offset_lod = ToAbsOffset(in);
  new_lod.back() = abs_offset_lod[level_end - 1];
  return new_lod;
}

LoD SliceInLevel(const LoD &in, size_t level, size_t elem_begin,
                 size_t elem_end) {
  PADDLE_ENFORCE_LT(level, in.size());
  PADDLE_ENFORCE_LT(elem_end, in[level].size());

  LoD res;
  res.resize(in.size() - level);
  // copy the first level
  res[0].assign(in[level].begin() + elem_begin,
                in[level].begin() + elem_end + 1);
  for (size_t lvl = 1; lvl < res.size(); lvl++) {
    const auto &in_level = in[level + lvl];
    const auto &above_level = res[lvl - 1];
    auto &out_level = res[lvl];
    out_level.assign(in_level.begin() + above_level.front(),
                     in_level.begin() + above_level.back() + 1);
  }
  for (size_t lvl = 0; lvl < res.size(); lvl++) {
    // to make the first offset equals 0, all the elements minus the first
    // element
    size_t front = res[lvl].front();
    for (auto &ele : res[lvl]) {
      ele -= front;
    }
  }
  return res;
}

LoD ToAbsOffset(const LoD &in) {
  // the lowest level stores relative offsets
  if (in.empty() || in.size() == 1) return in;
  LoD result = in;
  for (int level = result.size() - 2; level >= 0; level--) {
    for (auto &ele : result[level]) {
      ele = result[level + 1][ele];
    }
  }
  return result;
}

bool operator==(const LoD &a, const LoD &b) {
  if (a.size() != b.size()) {
    return false;
  }

  for (size_t i = 0; i < a.size(); i++) {
    const auto &a_level = a[i];
    const auto &b_level = b[i];
    if (a_level.size() != b_level.size()) {
      return false;
    }
    for (size_t j = 0; j < a_level.size(); j++) {
      if (a_level[j] != b_level[j]) {
        return false;
      }
    }
  }
  return true;
}

size_t LoDTensor::NumElements(size_t level, size_t idx) const {
  PADDLE_ENFORCE_LT(level, NumLevels());
  PADDLE_ENFORCE_LT(idx, NumElements(level));
  return lod_[level][idx + 1] - lod_[level][idx];
}

size_t LoDTensor::NumInstancesInElement(size_t level, size_t idx) const {
  PADDLE_ENFORCE_LT(level, NumLevels());
  PADDLE_ENFORCE_LT(idx, NumElements(level));
  auto abs_lod = ToAbsOffset(lod());
  size_t begin = abs_lod[level][idx];
  size_t end = abs_lod[level][idx + 1];
  return end - begin;
}

void LoDTensor::ShrinkLevels(size_t level_begin, size_t level_end) {
  auto new_lod = framework::SliceLevels(lod_, level_begin, level_end);
  lod_ = new_lod;
}

void LoDTensor::ShrinkInLevel(size_t level, size_t elem_begin,
                              size_t elem_end) {
  PADDLE_ENFORCE_LT(level, NumLevels());
  PADDLE_ENFORCE_LT(elem_begin, NumElements(level));
  PADDLE_ENFORCE_LT(elem_end, NumElements(level) + 1);

  auto abs_lod = framework::ToAbsOffset(lod());
  auto new_lod = framework::SliceInLevel(lod_, level, elem_begin, elem_end);
  lod_ = new_lod;

  // slice the underlying tensor
  size_t begin = abs_lod[level][elem_begin];
  size_t end = abs_lod[level][elem_end];
  PADDLE_ENFORCE_LT(begin, end, "Cannot shrink, the result tensor is empty.");
  ShareDataWith(Slice(begin, end));
}

using LoDAndOffset = std::pair<LoD, std::pair<size_t, size_t>>;
LoDAndOffset GetSubLoDAndAbsoluteOffset(const LoD &lod, size_t start_idx,
                                        size_t end_idx, size_t start_level) {
  LoD sub_lod;

  for (size_t level_idx = start_level; level_idx < lod.size(); ++level_idx) {
    PADDLE_ENFORCE_LE(start_idx, end_idx);
    PADDLE_ENFORCE_LT(end_idx, lod[level_idx].size());
    std::vector<size_t> level_lens;
    for (size_t i = start_idx; i < end_idx; ++i) {
      level_lens.push_back(lod[level_idx][i + 1] - lod[level_idx][i]);
    }
    sub_lod.emplace_back(level_lens);
    start_idx = lod[level_idx][start_idx];
    end_idx = lod[level_idx][end_idx];
  }

  return LoDAndOffset{sub_lod, {start_idx, end_idx}};
}

void AppendLoD(LoD *lod, const LoD &lod_length) {
  PADDLE_ENFORCE(
      lod->empty() || lod->size() == lod_length.size(),
      "The lod_length should has the same size with the appended lod.");
  if (lod->empty()) {
    *lod = LoD(lod_length.size(), std::vector<size_t>({0}));
  }
  for (size_t i = 0; i < lod->size(); ++i) {
    auto &level = (*lod)[i];
    for (size_t len : lod_length[i]) {
      level.push_back(level.back() + len);
    }
  }
}

void SerializeToStream(std::ostream &os, const LoDTensor &tensor,
                       const platform::DeviceContext &dev_ctx) {
  {  // the 1st field, uint32_t version for LoDTensor
    constexpr uint32_t version = 0;
    os.write(reinterpret_cast<const char *>(&version), sizeof(version));
  }
  {
    // the 2st field, LoD information
    // uint64_t lod_level
    // uint64_t lod_level_1 size in byte.
    // int*     lod_level_1 data
    // ...
    auto lod = tensor.lod();
    uint64_t size = lod.size();
    os.write(reinterpret_cast<const char *>(&size), sizeof(size));

    for (auto &each : lod) {
      size = each.size() * sizeof(framework::LoD::value_type::value_type);
      os.write(reinterpret_cast<const char *>(&size), sizeof(size));
      os.write(reinterpret_cast<const char *>(each.data()),
               static_cast<std::streamsize>(size));
    }
  }
  // the 3st field, Tensor
  SerializeToStream(os, static_cast<Tensor>(tensor), dev_ctx);
}

void DeserializeFromStream(std::istream &is, LoDTensor *tensor) {
  {
    // the 1st field, unit32_t version for SelectedRows
    uint32_t version;
    is.read(reinterpret_cast<char *>(&version), sizeof(version));
    PADDLE_ENFORCE_EQ(version, 0U, "Only version 0 is supported");
  }
  {
    // the 2st field, LoD information
    uint64_t lod_level;
    is.read(reinterpret_cast<char *>(&lod_level), sizeof(lod_level));
    auto &lod = *tensor->mutable_lod();
    lod.resize(lod_level);
    for (uint64_t i = 0; i < lod_level; ++i) {
      uint64_t size;
      is.read(reinterpret_cast<char *>(&size), sizeof(size));
      std::vector<size_t> tmp(size / sizeof(size_t));
      is.read(reinterpret_cast<char *>(tmp.data()),
              static_cast<std::streamsize>(size));
      lod[i] = tmp;
    }
  }
  // the 3st filed, Tensor
  DeserializeFromStream(is, static_cast<Tensor *>(tensor));
}

}  // namespace framework
}  // namespace paddle
<commit_msg>Update code<commit_after>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "paddle/framework/lod_tensor.h"
#include "paddle/framework/data_type.h"
#include "paddle/framework/framework.pb.h"

#include "paddle/memory/memcpy.h"
#include "paddle/memory/memory.h"

#include <stdint.h>
#include <string.h>
#include <algorithm>
#include <iterator>

#include <glog/logging.h>

namespace paddle {
namespace framework {

std::ostream &operator<<(std::ostream &os, const LoD &lod) {
  os << "{";
  for (auto &v : lod) {
    os << "{";
    for (auto &i : v) {
      os << i << ",";
    }
    os << "}";
  }
  os << "}";

  return os;
}

LoD SliceLevels(const LoD &in, size_t level_begin, size_t level_end) {
  LoD new_lod;
  new_lod.reserve(level_end - level_begin);
  for (size_t i = level_begin; i < level_end; i++) {
    new_lod.emplace_back(in.at(i));
  }
  // transform the lowest level to absolute offset.
  LoD abs_offset_lod = ToAbsOffset(in);
  new_lod.back() = abs_offset_lod[level_end - 1];
  return new_lod;
}

LoD SliceInLevel(const LoD &in, size_t level, size_t elem_begin,
                 size_t elem_end) {
  PADDLE_ENFORCE_LT(level, in.size());
  PADDLE_ENFORCE_LT(elem_end, in[level].size());

  LoD res;
  res.resize(in.size() - level);
  // copy the first level
  res[0].assign(in[level].begin() + elem_begin,
                in[level].begin() + elem_end + 1);
  for (size_t lvl = 1; lvl < res.size(); lvl++) {
    const auto &in_level = in[level + lvl];
    const auto &above_level = res[lvl - 1];
    auto &out_level = res[lvl];
    out_level.assign(in_level.begin() + above_level.front(),
                     in_level.begin() + above_level.back() + 1);
  }
  for (size_t lvl = 0; lvl < res.size(); lvl++) {
    // to make the first offset equals 0, all the elements minus the first
    // element
    size_t front = res[lvl].front();
    for (auto &ele : res[lvl]) {
      ele -= front;
    }
  }
  return res;
}

LoD ToAbsOffset(const LoD &in) {
  // the lowest level stores relative offsets
  if (in.empty() || in.size() == 1) return in;
  LoD result = in;
  for (int level = result.size() - 2; level >= 0; level--) {
    for (auto &ele : result[level]) {
      ele = result[level + 1][ele];
    }
  }
  return result;
}

bool operator==(const LoD &a, const LoD &b) {
  if (a.size() != b.size()) {
    return false;
  }

  for (size_t i = 0; i < a.size(); i++) {
    const auto &a_level = a[i];
    const auto &b_level = b[i];
    if (a_level.size() != b_level.size()) {
      return false;
    }
    for (size_t j = 0; j < a_level.size(); j++) {
      if (a_level[j] != b_level[j]) {
        return false;
      }
    }
  }
  return true;
}

size_t LoDTensor::NumElements(size_t level, size_t idx) const {
  PADDLE_ENFORCE_LT(level, NumLevels());
  PADDLE_ENFORCE_LT(idx, NumElements(level));
  return lod_[level][idx + 1] - lod_[level][idx];
}

size_t LoDTensor::NumInstancesInElement(size_t level, size_t idx) const {
  PADDLE_ENFORCE_LT(level, NumLevels());
  PADDLE_ENFORCE_LT(idx, NumElements(level));
  auto abs_lod = ToAbsOffset(lod());
  size_t begin = abs_lod[level][idx];
  size_t end = abs_lod[level][idx + 1];
  return end - begin;
}

void LoDTensor::ShrinkLevels(size_t level_begin, size_t level_end) {
  auto new_lod = framework::SliceLevels(lod_, level_begin, level_end);
  lod_ = new_lod;
}

void LoDTensor::ShrinkInLevel(size_t level, size_t elem_begin,
                              size_t elem_end) {
  PADDLE_ENFORCE_LT(level, NumLevels());
  PADDLE_ENFORCE_LT(elem_begin, NumElements(level));
  PADDLE_ENFORCE_LT(elem_end, NumElements(level) + 1);

  auto abs_lod = framework::ToAbsOffset(lod());
  auto new_lod = framework::SliceInLevel(lod_, level, elem_begin, elem_end);
  lod_ = new_lod;

  // slice the underlying tensor
  size_t begin = abs_lod[level][elem_begin];
  size_t end = abs_lod[level][elem_end];
  PADDLE_ENFORCE_LT(begin, end, "Cannot shrink, the result tensor is empty.");
  ShareDataWith(Slice(begin, end));
}

using LoDAndOffset = std::pair<LoD, std::pair<size_t, size_t>>;
LoDAndOffset GetSubLoDAndAbsoluteOffset(const LoD &lod, size_t start_idx,
                                        size_t end_idx, size_t start_level) {
  LoD sub_lod;

  for (size_t level_idx = start_level; level_idx < lod.size(); ++level_idx) {
    PADDLE_ENFORCE_LE(start_idx, end_idx);
    PADDLE_ENFORCE_LT(end_idx, lod[level_idx].size());
    std::vector<size_t> level_lens;
    for (size_t i = start_idx; i < end_idx; ++i) {
      level_lens.push_back(lod[level_idx][i + 1] - lod[level_idx][i]);
    }
    sub_lod.emplace_back(level_lens);
    start_idx = lod[level_idx][start_idx];
    end_idx = lod[level_idx][end_idx];
  }

  return LoDAndOffset{sub_lod, {start_idx, end_idx}};
}

void AppendLoD(LoD *lod, const LoD &lod_length) {
  PADDLE_ENFORCE(
      lod->empty() || lod->size() == lod_length.size(),
      "The lod_length should has the same size with the appended lod.");
  if (lod->empty()) {
    for (size_t i = 0; i < lod_length.size(); ++i) {
      lod->emplace_back(1, 0);  // size = 1, value = 0;
    }
    *lod = LoD(lod_length.size(), std::vector<size_t>({0}));
  }
  for (size_t i = 0; i < lod->size(); ++i) {
    auto &level = (*lod)[i];
    for (size_t len : lod_length[i]) {
      level.push_back(level.back() + len);
    }
  }
}

void SerializeToStream(std::ostream &os, const LoDTensor &tensor,
                       const platform::DeviceContext &dev_ctx) {
  {  // the 1st field, uint32_t version for LoDTensor
    constexpr uint32_t version = 0;
    os.write(reinterpret_cast<const char *>(&version), sizeof(version));
  }
  {
    // the 2st field, LoD information
    // uint64_t lod_level
    // uint64_t lod_level_1 size in byte.
    // int*     lod_level_1 data
    // ...
    auto lod = tensor.lod();
    uint64_t size = lod.size();
    os.write(reinterpret_cast<const char *>(&size), sizeof(size));

    for (auto &each : lod) {
      size = each.size() * sizeof(framework::LoD::value_type::value_type);
      os.write(reinterpret_cast<const char *>(&size), sizeof(size));
      os.write(reinterpret_cast<const char *>(each.data()),
               static_cast<std::streamsize>(size));
    }
  }
  // the 3st field, Tensor
  SerializeToStream(os, static_cast<Tensor>(tensor), dev_ctx);
}

void DeserializeFromStream(std::istream &is, LoDTensor *tensor) {
  {
    // the 1st field, unit32_t version for SelectedRows
    uint32_t version;
    is.read(reinterpret_cast<char *>(&version), sizeof(version));
    PADDLE_ENFORCE_EQ(version, 0U, "Only version 0 is supported");
  }
  {
    // the 2st field, LoD information
    uint64_t lod_level;
    is.read(reinterpret_cast<char *>(&lod_level), sizeof(lod_level));
    auto &lod = *tensor->mutable_lod();
    lod.resize(lod_level);
    for (uint64_t i = 0; i < lod_level; ++i) {
      uint64_t size;
      is.read(reinterpret_cast<char *>(&size), sizeof(size));
      std::vector<size_t> tmp(size / sizeof(size_t));
      is.read(reinterpret_cast<char *>(tmp.data()),
              static_cast<std::streamsize>(size));
      lod[i] = tmp;
    }
  }
  // the 3st filed, Tensor
  DeserializeFromStream(is, static_cast<Tensor *>(tensor));
}

}  // namespace framework
}  // namespace paddle
<|endoftext|>


<commit_before>#include <boost/make_shared.hpp>
#include <ompl/base/ScopedState.h>
#include "OMPLConversions.h"
#include "OMPLSimplifer.h"

namespace or_ompl
{

OMPLSimplifier::OMPLSimplifier(OpenRAVE::EnvironmentBasePtr env)
    : OpenRAVE::PlannerBase(env)
{
}

OMPLSimplifier::~OMPLSimplifier()
{
}

bool OMPLSimplifier::InitPlan(OpenRAVE::RobotBasePtr robot,
                              PlannerParametersConstPtr params_raw)
{
    if (!robot) {
        RAVELOG_ERROR("Robot must not be NULL.\n");
        return false;
    } else if (!params_raw) {
        RAVELOG_ERROR("Parameters must not be NULL.\n");
        return false;
    }

    m_robot = robot;
    m_cspec = m_robot->GetActiveConfigurationSpecification();

    m_parameters = boost::make_shared<OMPLPlannerParameters>();
    m_parameters->copy(params_raw);

    try {
        using ompl::base::SpaceInformation;
        using ompl::geometric::PathSimplifier;

        m_state_space = CreateStateSpace(robot, *m_parameters);
        m_space_info = boost::make_shared<SpaceInformation>(m_state_space);
        m_space_info->setStateValidityChecker(
                boost::bind(&OMPLSimplifier::IsStateValid, this, _1));
        m_simplifier = boost::make_shared<PathSimplifier>(m_space_info);
        return true;
    } catch (std::runtime_error const &e) {
        RAVELOG_ERROR("IntPlan failed: %s\n", e.what());
        return false;
    }
}

bool OMPLSimplifier::InitPlan(OpenRAVE::RobotBasePtr robot, std::istream &input)
{
    OMPLPlannerParametersPtr params = boost::make_shared<OMPLPlannerParameters>();
    input >> *params;
    return InitPlan(robot, params);
}

OpenRAVE::PlannerStatus OMPLSimplifier::PlanPath(OpenRAVE::TrajectoryBasePtr ptraj)
{
    typedef ompl::base::RealVectorStateSpace::StateType StateType;
    typedef ompl::base::ScopedState<ompl::base::RealVectorStateSpace> ScopedState;

    if (!m_simplifier) {
        RAVELOG_ERROR("Not initialized. Did you call InitPlan?\n");
        return OpenRAVE::PS_Failed;
    } else if (!ptraj) {
        RAVELOG_ERROR("Input trajectory is NULL.\n");
        return OpenRAVE::PS_Failed;
    } else if (m_parameters->m_timeLimit <= 0) {
        RAVELOG_ERROR("Time limit must be positive; got %f.",
                      m_parameters->m_timeLimit);
        return OpenRAVE::PS_Failed;
    } else if (ptraj->GetNumWaypoints() == 0) {
        RAVELOG_WARN("Input trajectory is empty; there is nothing to do.\n");
        return OpenRAVE::PS_HasSolution;
    }

    // Convert the OpenRAVE trajectory into an OMPL path.
    BOOST_ASSERT(m_space_info);
    ompl::geometric::PathGeometric path(m_space_info);
    size_t const num_dof = m_cspec.GetDOF();

    RAVELOG_DEBUG("Create OMPL path with %d DOF and %d waypoints.\n",
                  num_dof, ptraj->GetNumWaypoints());

    for (size_t iwaypoint = 0; iwaypoint < ptraj->GetNumWaypoints(); ++iwaypoint) {
        // Extract the OpenRAVE waypoint. Default to the current configuration
        // for any missing DOFs.
        std::vector<OpenRAVE::dReal> waypoint_openrave;
        m_robot->GetActiveDOFValues(waypoint_openrave);
        ptraj->GetWaypoint(iwaypoint, waypoint_openrave, m_cspec);

        // Insert the waypoint into the OMPL path.
        ScopedState waypoint_ompl(m_space_info);
        for (size_t idof = 0; idof < num_dof; ++idof) {
            waypoint_ompl[idof] = waypoint_openrave[idof];
        }
        path.append(waypoint_ompl.get());
    }

    // Run path simplification.
    // TODO: Why does this SEGFAULT? Something must be wrong with the OMPL path
    RAVELOG_DEBUG("Running path simplification for %f seconds.\n",
                  m_parameters->m_timeLimit);
    BOOST_ASSERT(m_parameters);
    m_simplifier->simplify(path, m_parameters->m_timeLimit);

    // Store the result in the OpenRAVE trajectory.
    RAVELOG_DEBUG("Reconstructing OpenRAVE trajectory with %d waypoints.\n",
                  path.getStateCount());
    BOOST_ASSERT(ptraj);
    ptraj->Remove(0, ptraj->GetNumWaypoints());

    for (size_t iwaypoint = 0; iwaypoint < path.getStateCount(); ++iwaypoint) {
        ompl::base::State const *waypoint_generic = path.getState(iwaypoint);
        StateType const &waypoint_ompl = *waypoint_generic->as<StateType>();

        std::vector<OpenRAVE::dReal> waypoint_openrave(num_dof);
        for (size_t idof = 0; idof < num_dof; ++idof) {
            waypoint_openrave[idof] = waypoint_ompl[idof];
        }
        ptraj->Insert(iwaypoint, waypoint_openrave, m_cspec);
    }
    return OpenRAVE::PS_HasSolution;
}

bool OMPLSimplifier::IsInOrCollision(std::vector<double> const &values)
{
    m_robot->SetActiveDOFValues(values, OpenRAVE::KinBody::CLA_Nothing);
    return GetEnv()->CheckCollision(m_robot) || m_robot->CheckSelfCollision();
}

bool OMPLSimplifier::IsStateValid(ompl::base::State const *state)
{
    typedef ompl::base::RealVectorStateSpace::StateType StateType;
    StateType const *realVectorState = state->as<StateType>();
    size_t const num_dof = m_robot->GetActiveDOF();

    if (realVectorState) {
        std::vector<OpenRAVE::dReal> values(num_dof);
        for (size_t i = 0; i < num_dof; i++) {
            values[i] = realVectorState->values[i];
        }
        return !IsInOrCollision(values);
    } else {
        RAVELOG_ERROR("Invalid StateType. This should never happen.\n");
        return false;
    }
}

}
<commit_msg>Call setup() on SpaceInformation in the simplifier<commit_after>#include <boost/make_shared.hpp>
#include <ompl/base/ScopedState.h>
#include "OMPLConversions.h"
#include "OMPLSimplifer.h"

namespace or_ompl
{

OMPLSimplifier::OMPLSimplifier(OpenRAVE::EnvironmentBasePtr env)
    : OpenRAVE::PlannerBase(env)
{
}

OMPLSimplifier::~OMPLSimplifier()
{
}

bool OMPLSimplifier::InitPlan(OpenRAVE::RobotBasePtr robot,
                              PlannerParametersConstPtr params_raw)
{
    if (!robot) {
        RAVELOG_ERROR("Robot must not be NULL.\n");
        return false;
    } else if (!params_raw) {
        RAVELOG_ERROR("Parameters must not be NULL.\n");
        return false;
    }

    m_robot = robot;
    m_cspec = m_robot->GetActiveConfigurationSpecification();

    m_parameters = boost::make_shared<OMPLPlannerParameters>();
    m_parameters->copy(params_raw);

    try {
        using ompl::base::SpaceInformation;
        using ompl::geometric::PathSimplifier;

        m_state_space = CreateStateSpace(robot, *m_parameters);
        m_space_info = boost::make_shared<SpaceInformation>(m_state_space);
        m_space_info->setStateValidityChecker(
                boost::bind(&OMPLSimplifier::IsStateValid, this, _1));
        m_space_info->setup();
        m_simplifier = boost::make_shared<PathSimplifier>(m_space_info);
        return true;
    } catch (std::runtime_error const &e) {
        RAVELOG_ERROR("IntPlan failed: %s\n", e.what());
        return false;
    }
}

bool OMPLSimplifier::InitPlan(OpenRAVE::RobotBasePtr robot, std::istream &input)
{
    OMPLPlannerParametersPtr params = boost::make_shared<OMPLPlannerParameters>();
    input >> *params;
    return InitPlan(robot, params);
}

OpenRAVE::PlannerStatus OMPLSimplifier::PlanPath(OpenRAVE::TrajectoryBasePtr ptraj)
{
    typedef ompl::base::RealVectorStateSpace::StateType StateType;
    typedef ompl::base::ScopedState<ompl::base::RealVectorStateSpace> ScopedState;

    if (!m_simplifier) {
        RAVELOG_ERROR("Not initialized. Did you call InitPlan?\n");
        return OpenRAVE::PS_Failed;
    } else if (!ptraj) {
        RAVELOG_ERROR("Input trajectory is NULL.\n");
        return OpenRAVE::PS_Failed;
    } else if (m_parameters->m_timeLimit <= 0) {
        RAVELOG_ERROR("Time limit must be positive; got %f.",
                      m_parameters->m_timeLimit);
        return OpenRAVE::PS_Failed;
    } else if (ptraj->GetNumWaypoints() == 0) {
        RAVELOG_WARN("Input trajectory is empty; there is nothing to do.\n");
        return OpenRAVE::PS_HasSolution;
    }

    // Convert the OpenRAVE trajectory into an OMPL path.
    BOOST_ASSERT(m_space_info);
    ompl::geometric::PathGeometric path(m_space_info);
    size_t const num_dof = m_cspec.GetDOF();

    RAVELOG_DEBUG("Create OMPL path with %d DOF and %d waypoints.\n",
                  num_dof, ptraj->GetNumWaypoints());

    for (size_t iwaypoint = 0; iwaypoint < ptraj->GetNumWaypoints(); ++iwaypoint) {
        // Extract the OpenRAVE waypoint. Default to the current configuration
        // for any missing DOFs.
        std::vector<OpenRAVE::dReal> waypoint_openrave;
        m_robot->GetActiveDOFValues(waypoint_openrave);
        ptraj->GetWaypoint(iwaypoint, waypoint_openrave, m_cspec);

        // Insert the waypoint into the OMPL path.
        ScopedState waypoint_ompl(m_space_info);
        for (size_t idof = 0; idof < num_dof; ++idof) {
            waypoint_ompl[idof] = waypoint_openrave[idof];
        }
        path.append(waypoint_ompl.get());
    }

    // Run path simplification.
    // TODO: Why does this SEGFAULT? Something must be wrong with the OMPL path
    RAVELOG_DEBUG("Running path simplification for %f seconds.\n",
                  m_parameters->m_timeLimit);
    BOOST_ASSERT(m_parameters);
    m_simplifier->simplify(path, m_parameters->m_timeLimit);

    // Store the result in the OpenRAVE trajectory.
    RAVELOG_DEBUG("Reconstructing OpenRAVE trajectory with %d waypoints.\n",
                  path.getStateCount());
    BOOST_ASSERT(ptraj);
    ptraj->Remove(0, ptraj->GetNumWaypoints());

    for (size_t iwaypoint = 0; iwaypoint < path.getStateCount(); ++iwaypoint) {
        ompl::base::State const *waypoint_generic = path.getState(iwaypoint);
        StateType const &waypoint_ompl = *waypoint_generic->as<StateType>();

        std::vector<OpenRAVE::dReal> waypoint_openrave(num_dof);
        for (size_t idof = 0; idof < num_dof; ++idof) {
            waypoint_openrave[idof] = waypoint_ompl[idof];
        }
        ptraj->Insert(iwaypoint, waypoint_openrave, m_cspec);
    }
    return OpenRAVE::PS_HasSolution;
}

bool OMPLSimplifier::IsInOrCollision(std::vector<double> const &values)
{
    m_robot->SetActiveDOFValues(values, OpenRAVE::KinBody::CLA_Nothing);
    return GetEnv()->CheckCollision(m_robot) || m_robot->CheckSelfCollision();
}

bool OMPLSimplifier::IsStateValid(ompl::base::State const *state)
{
    typedef ompl::base::RealVectorStateSpace::StateType StateType;
    StateType const *realVectorState = state->as<StateType>();
    size_t const num_dof = m_robot->GetActiveDOF();

    if (realVectorState) {
        std::vector<OpenRAVE::dReal> values(num_dof);
        for (size_t i = 0; i < num_dof; i++) {
            values[i] = realVectorState->values[i];
        }
        return !IsInOrCollision(values);
    } else {
        RAVELOG_ERROR("Invalid StateType. This should never happen.\n");
        return false;
    }
}

}
<|endoftext|>


<commit_before>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE-CHROMIUM file.

#include "browser_client.h"

#include "browser_context.h"
#include "browser_main_parts.h"
#include "notification_presenter.h"

namespace brightray {

namespace {

BrowserClient* g_browser_client;

}

BrowserClient* BrowserClient::Get() {
  return g_browser_client;
}

BrowserClient::BrowserClient()
    : browser_main_parts_() {
  DCHECK(!g_browser_client);
  g_browser_client = this;
}

BrowserClient::~BrowserClient() {
}

BrowserContext* BrowserClient::browser_context() {
  return browser_main_parts_->browser_context();
}

NotificationPresenter* BrowserClient::notification_presenter() {
#if defined(OS_MACOSX)
  if (!notification_presenter_)
    notification_presenter_.reset(NotificationPresenter::Create());
#endif
  return notification_presenter_.get();
}

BrowserMainParts* BrowserClient::OverrideCreateBrowserMainParts(const content::MainFunctionParams&) {
  return new BrowserMainParts;
}

content::BrowserMainParts* BrowserClient::CreateBrowserMainParts(const content::MainFunctionParams& parameters) {
  DCHECK(!browser_main_parts_);
  browser_main_parts_ = OverrideCreateBrowserMainParts(parameters);
  return browser_main_parts_;
}

net::URLRequestContextGetter* BrowserClient::CreateRequestContext(content::BrowserContext* browser_context, content::ProtocolHandlerMap* protocol_handlers) {
  return static_cast<BrowserContext*>(browser_context)->CreateRequestContext(protocol_handlers);
}

void BrowserClient::ShowDesktopNotification(
    const content::ShowDesktopNotificationHostMsgParams& params,
    int render_process_id,
    int render_view_id,
    bool worker) {
  notification_presenter()->ShowNotification(params, render_process_id, render_view_id);
}

void BrowserClient::CancelDesktopNotification(
    int render_process_id,
    int render_view_id,
    int notification_id) {
  notification_presenter()->CancelNotification(render_process_id, render_view_id, notification_id);
}

}
<commit_msg>Don't crash when HTML notifications are used on Windows<commit_after>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE-CHROMIUM file.

#include "browser_client.h"

#include "browser_context.h"
#include "browser_main_parts.h"
#include "notification_presenter.h"

namespace brightray {

namespace {

BrowserClient* g_browser_client;

}

BrowserClient* BrowserClient::Get() {
  return g_browser_client;
}

BrowserClient::BrowserClient()
    : browser_main_parts_() {
  DCHECK(!g_browser_client);
  g_browser_client = this;
}

BrowserClient::~BrowserClient() {
}

BrowserContext* BrowserClient::browser_context() {
  return browser_main_parts_->browser_context();
}

NotificationPresenter* BrowserClient::notification_presenter() {
#if defined(OS_MACOSX)
  if (!notification_presenter_)
    notification_presenter_.reset(NotificationPresenter::Create());
#endif
  return notification_presenter_.get();
}

BrowserMainParts* BrowserClient::OverrideCreateBrowserMainParts(const content::MainFunctionParams&) {
  return new BrowserMainParts;
}

content::BrowserMainParts* BrowserClient::CreateBrowserMainParts(const content::MainFunctionParams& parameters) {
  DCHECK(!browser_main_parts_);
  browser_main_parts_ = OverrideCreateBrowserMainParts(parameters);
  return browser_main_parts_;
}

net::URLRequestContextGetter* BrowserClient::CreateRequestContext(content::BrowserContext* browser_context, content::ProtocolHandlerMap* protocol_handlers) {
  return static_cast<BrowserContext*>(browser_context)->CreateRequestContext(protocol_handlers);
}

void BrowserClient::ShowDesktopNotification(
    const content::ShowDesktopNotificationHostMsgParams& params,
    int render_process_id,
    int render_view_id,
    bool worker) {
  auto presenter = notification_presenter();
  if (!presenter)
    return;
  presenter->ShowNotification(params, render_process_id, render_view_id);
}

void BrowserClient::CancelDesktopNotification(
    int render_process_id,
    int render_view_id,
    int notification_id) {
  auto presenter = notification_presenter();
  if (!presenter)
    return;
  presenter->CancelNotification(render_process_id, render_view_id, notification_id);
}

}
<|endoftext|>