<commit_before>/*
* Copyright  2015 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/

#include "stdafx.h"
#include "ExploreClient.h"
#include "BadRequestException.h"
#include "CdapException.h"

using namespace Cask::CdapOdbc;

namespace {
  web::json::value toJson(const utility::string_t* str) {
    return (str != nullptr) ? web::json::value::string(*str) : web::json::value::null();
  }

  utility::string_t toUriPath(const utility::string_t* str) {
    if (str == nullptr) {
      return L"null";
    } else {
      return *str;
    }
  }
}

web::json::value Cask::CdapOdbc::ExploreClient::doRequest(web::http::http_request& request, std::int64_t* sizeInBytes) {
#ifdef TRACE_REQUESTS
  TRACE(L"REQUEST: %s\n", request.to_string().c_str());
#endif
  auto requestTask = this->httpClient->request(request);
  {
    PROFILE_FUNCTION(TIMER_QUERY);
    requestTask.wait();
  }

  auto response = requestTask.get();
#ifdef TRACE_REQUESTS
  TRACE(L"RESPONSE: %s\n", response.to_string().c_str());
#endif
//  TRACE(L"RESPONSE SIZE: %d\n", response.headers().content_length());
  if (response.status_code() == web::http::status_codes::OK) {
    auto jsonTask = response.extract_json();
    {
      PROFILE_FUNCTION(TIMER_PARSING);
      jsonTask.wait();
    }

    if (sizeInBytes) {
      *sizeInBytes = response.headers().content_length();
    }

    return jsonTask.get();
  } else if (response.status_code() == web::http::status_codes::BadRequest) {
    auto errorTask = response.extract_utf16string();
    errorTask.wait();
    throw BadRequestException(errorTask.get());
  } else {
    throw CdapException(L"Cannot get the response.");
  }
}

web::json::value Cask::CdapOdbc::ExploreClient::doRequest(web::http::method mhd, const utility::string_t& path, const web::json::value* body /* = nullptr */, std::int64_t* sizeInBytes /* = nullptr */) {
  web::http::uri_builder requestUri;
  web::http::http_request request(mhd);

  if (body) {
    request.set_body(*body);
  }

  request.set_request_uri(requestUri.append_path(path).to_uri());
  if (this->authToken.size() > 0) {
    request.headers().add(web::http::header_names::authorization, L"Bearer " + this->authToken);
  }

  return this->doRequest(request, sizeInBytes);
}

Cask::CdapOdbc::ExploreClient::ExploreClient(const web::http::uri& baseUri, const std::wstring& namespace_, const SecureString& authToken)
  : namespace_(namespace_)
  , authToken(authToken) {
  this->httpClient = std::make_unique<web::http::client::http_client>(baseUri);
}

bool Cask::CdapOdbc::ExploreClient::isAvailable() {
  try {
    return (this->doRequest(web::http::methods::GET, L"explore/status").at(L"status").as_string() == L"OK");
  } catch (std::exception&) {
    return false;
  }
}

QueryStatus Cask::CdapOdbc::ExploreClient::getQueryStatus(const QueryHandle& handle) {
  auto value = this->doRequest(web::http::methods::GET, L"data/explore/queries/" + handle + L"/status");
  return QueryStatus(value);
}

std::vector<ColumnDesc> Cask::CdapOdbc::ExploreClient::getQuerySchema(const QueryHandle& handle) {
  std::vector<ColumnDesc> result;
  auto value = this->doRequest(web::http::methods::GET, L"data/explore/queries/" + handle + L"/schema");
  auto columns = value.as_array();
  for (auto& item : columns) {
    result.push_back(ColumnDesc(item));
  }

  return result;
}

QueryResult Cask::CdapOdbc::ExploreClient::getQueryResult(const QueryHandle& handle, int rows) {
  web::json::value size;
  std::int64_t sizeInBytes = 0L;
  size[L"size"] = web::json::value::number(rows);
  auto value = this->doRequest(web::http::methods::POST, L"data/explore/queries/" + handle + L"/next", &size, &sizeInBytes);
  return QueryResult(value, sizeInBytes);
}

void Cask::CdapOdbc::ExploreClient::closeQuery(const QueryHandle& handle) {
  this->doRequest(web::http::methods::DEL, L"data/explore/queries/" + handle);
}

QueryHandle Cask::CdapOdbc::ExploreClient::getCatalogs() {
  return this->doRequest(web::http::methods::POST, L"data/explore/jdbc/catalogs").at(L"handle").as_string();
}

QueryHandle Cask::CdapOdbc::ExploreClient::getSchemas(const std::wstring* catalog, const std::wstring* schemaPattern) {
  web::json::value schemaArgs;
  schemaArgs[L"catalog"] = toJson(catalog);
  schemaArgs[L"schemaPattern"] = toJson(schemaPattern);

  auto value = this->doRequest(web::http::methods::POST, L"namespaces/" + toUriPath(schemaPattern) + L"/data/explore/jdbc/schemas", &schemaArgs);
  return value.at(L"handle").as_string();
}

QueryHandle Cask::CdapOdbc::ExploreClient::getTables(const std::wstring* catalog, const std::wstring* schemaPattern, const std::wstring* tableNamePattern, const std::vector<std::wstring>* tableTypes) {
  web::json::value tablesArgs;

  tablesArgs[L"catalog"] = toJson(catalog);
  tablesArgs[L"schemaPattern"] = toJson(schemaPattern);
  tablesArgs[L"tableNamePattern"] = toJson(tableNamePattern);

  if (tableTypes) {
    std::vector<web::json::value> tableTypeArgs;

    for (auto& item : *tableTypes) {
      tableTypeArgs.push_back(web::json::value::string(item));
    }

    tablesArgs[L"tableTypes"] = web::json::value::array(tableTypeArgs);
  } else {
    tablesArgs[L"tableTypes"] = web::json::value::null();
  }

  auto value = this->doRequest(web::http::methods::POST, L"namespaces/" + toUriPath(schemaPattern) + L"/data/explore/jdbc/tables", &tablesArgs);
  return value.at(L"handle").as_string();
}

QueryResult Cask::CdapOdbc::ExploreClient::getStreams() {
  auto value = this->doRequest(web::http::methods::GET, L"namespaces/" + this->namespace_ + L"/streams");
  return QueryResult(value);
}

QueryResult Cask::CdapOdbc::ExploreClient::getDatasets() {
  auto value = this->doRequest(web::http::methods::GET, L"namespaces/" + this->namespace_ + L"/data/datasets");
  return QueryResult(value);
}

QueryResult Cask::CdapOdbc::ExploreClient::getStreamFields(const std::wstring& streamName) {
  auto value = this->doRequest(web::http::methods::GET, L"namespaces/" + this->namespace_ + L"/streams/" + streamName);
  return QueryResult(value);
}

QueryResult Cask::CdapOdbc::ExploreClient::getDatasetFields(const std::wstring& datasetName) {
  auto value = this->doRequest(web::http::methods::GET, L"namespaces/" + this->namespace_ + L"/data/datasets/" + datasetName);
  return QueryResult(value);
}

QueryHandle Cask::CdapOdbc::ExploreClient::execute(const std::wstring& statement) {
  web::json::value query;
  std::wstring stmt_preproc = statement;
  std::wstring::size_type found;

  /* Remove 'HAVING (COUNT(1) > 0) clause' */
  found = stmt_preproc.find(L"HAVING (COUNT(1) > 0)", 0);
  if (found != std::wstring::npos) {
    stmt_preproc.erase(found, 21);
  }

  query[L"query"] = toJson(&stmt_preproc);
  auto value = this->doRequest(web::http::methods::POST, L"namespaces/" + this->namespace_ + L"/data/explore/queries", &query);
  return value.at(L"handle").as_string();
}
<commit_msg>Handling of unathorized response.<commit_after>/*
* Copyright  2015 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/

#include "stdafx.h"
#include "ExploreClient.h"
#include "BadRequestException.h"
#include "CdapException.h"
#include "CommunicationLinkFailure.h"

using namespace Cask::CdapOdbc;

namespace {
  web::json::value toJson(const utility::string_t* str) {
    return (str != nullptr) ? web::json::value::string(*str) : web::json::value::null();
  }

  utility::string_t toUriPath(const utility::string_t* str) {
    if (str == nullptr) {
      return L"null";
    } else {
      return *str;
    }
  }
}

web::json::value Cask::CdapOdbc::ExploreClient::doRequest(web::http::http_request& request, std::int64_t* sizeInBytes) {
#ifdef TRACE_REQUESTS
  TRACE(L"REQUEST: %s\n", request.to_string().c_str());
#endif
  auto requestTask = this->httpClient->request(request);
  {
    PROFILE_FUNCTION(TIMER_QUERY);
    requestTask.wait();
  }

  auto response = requestTask.get();
#ifdef TRACE_REQUESTS
  TRACE(L"RESPONSE: %s\n", response.to_string().c_str());
#endif
//  TRACE(L"RESPONSE SIZE: %d\n", response.headers().content_length());
  if (response.status_code() == web::http::status_codes::OK) {
    auto jsonTask = response.extract_json();
    {
      PROFILE_FUNCTION(TIMER_PARSING);
      jsonTask.wait();
    }

    if (sizeInBytes) {
      *sizeInBytes = response.headers().content_length();
    }

    return jsonTask.get();
  } else if (response.status_code() == web::http::status_codes::BadRequest) {
    auto errorTask = response.extract_utf16string();
    errorTask.wait();
    throw BadRequestException(errorTask.get());
  } else if (response.status_code() == web::http::status_codes::Unauthorized) {
    throw CdapException(L"Wrong authentication token.");
  } else {
    throw CommunicationLinkFailure();
  }
}

web::json::value Cask::CdapOdbc::ExploreClient::doRequest(web::http::method mhd, const utility::string_t& path, const web::json::value* body /* = nullptr */, std::int64_t* sizeInBytes /* = nullptr */) {
  web::http::uri_builder requestUri;
  web::http::http_request request(mhd);

  if (body) {
    request.set_body(*body);
  }

  request.set_request_uri(requestUri.append_path(path).to_uri());
  if (this->authToken.size() > 0) {
    request.headers().add(web::http::header_names::authorization, L"Bearer " + this->authToken);
  }

  return this->doRequest(request, sizeInBytes);
}

Cask::CdapOdbc::ExploreClient::ExploreClient(const web::http::uri& baseUri, const std::wstring& namespace_, const SecureString& authToken)
  : namespace_(namespace_)
  , authToken(authToken) {
  this->httpClient = std::make_unique<web::http::client::http_client>(baseUri);
}

bool Cask::CdapOdbc::ExploreClient::isAvailable() {
  return (this->doRequest(web::http::methods::GET, L"explore/status").at(L"status").as_string() == L"OK");
}

QueryStatus Cask::CdapOdbc::ExploreClient::getQueryStatus(const QueryHandle& handle) {
  auto value = this->doRequest(web::http::methods::GET, L"data/explore/queries/" + handle + L"/status");
  return QueryStatus(value);
}

std::vector<ColumnDesc> Cask::CdapOdbc::ExploreClient::getQuerySchema(const QueryHandle& handle) {
  std::vector<ColumnDesc> result;
  auto value = this->doRequest(web::http::methods::GET, L"data/explore/queries/" + handle + L"/schema");
  auto columns = value.as_array();
  for (auto& item : columns) {
    result.push_back(ColumnDesc(item));
  }

  return result;
}

QueryResult Cask::CdapOdbc::ExploreClient::getQueryResult(const QueryHandle& handle, int rows) {
  web::json::value size;
  std::int64_t sizeInBytes = 0L;
  size[L"size"] = web::json::value::number(rows);
  auto value = this->doRequest(web::http::methods::POST, L"data/explore/queries/" + handle + L"/next", &size, &sizeInBytes);
  return QueryResult(value, sizeInBytes);
}

void Cask::CdapOdbc::ExploreClient::closeQuery(const QueryHandle& handle) {
  this->doRequest(web::http::methods::DEL, L"data/explore/queries/" + handle);
}

QueryHandle Cask::CdapOdbc::ExploreClient::getCatalogs() {
  return this->doRequest(web::http::methods::POST, L"data/explore/jdbc/catalogs").at(L"handle").as_string();
}

QueryHandle Cask::CdapOdbc::ExploreClient::getSchemas(const std::wstring* catalog, const std::wstring* schemaPattern) {
  web::json::value schemaArgs;
  schemaArgs[L"catalog"] = toJson(catalog);
  schemaArgs[L"schemaPattern"] = toJson(schemaPattern);

  auto value = this->doRequest(web::http::methods::POST, L"namespaces/" + toUriPath(schemaPattern) + L"/data/explore/jdbc/schemas", &schemaArgs);
  return value.at(L"handle").as_string();
}

QueryHandle Cask::CdapOdbc::ExploreClient::getTables(const std::wstring* catalog, const std::wstring* schemaPattern, const std::wstring* tableNamePattern, const std::vector<std::wstring>* tableTypes) {
  web::json::value tablesArgs;

  tablesArgs[L"catalog"] = toJson(catalog);
  tablesArgs[L"schemaPattern"] = toJson(schemaPattern);
  tablesArgs[L"tableNamePattern"] = toJson(tableNamePattern);

  if (tableTypes) {
    std::vector<web::json::value> tableTypeArgs;

    for (auto& item : *tableTypes) {
      tableTypeArgs.push_back(web::json::value::string(item));
    }

    tablesArgs[L"tableTypes"] = web::json::value::array(tableTypeArgs);
  } else {
    tablesArgs[L"tableTypes"] = web::json::value::null();
  }

  auto value = this->doRequest(web::http::methods::POST, L"namespaces/" + toUriPath(schemaPattern) + L"/data/explore/jdbc/tables", &tablesArgs);
  return value.at(L"handle").as_string();
}

QueryResult Cask::CdapOdbc::ExploreClient::getStreams() {
  auto value = this->doRequest(web::http::methods::GET, L"namespaces/" + this->namespace_ + L"/streams");
  return QueryResult(value);
}

QueryResult Cask::CdapOdbc::ExploreClient::getDatasets() {
  auto value = this->doRequest(web::http::methods::GET, L"namespaces/" + this->namespace_ + L"/data/datasets");
  return QueryResult(value);
}

QueryResult Cask::CdapOdbc::ExploreClient::getStreamFields(const std::wstring& streamName) {
  auto value = this->doRequest(web::http::methods::GET, L"namespaces/" + this->namespace_ + L"/streams/" + streamName);
  return QueryResult(value);
}

QueryResult Cask::CdapOdbc::ExploreClient::getDatasetFields(const std::wstring& datasetName) {
  auto value = this->doRequest(web::http::methods::GET, L"namespaces/" + this->namespace_ + L"/data/datasets/" + datasetName);
  return QueryResult(value);
}

QueryHandle Cask::CdapOdbc::ExploreClient::execute(const std::wstring& statement) {
  web::json::value query;
  std::wstring stmt_preproc = statement;
  std::wstring::size_type found;

  /* Remove 'HAVING (COUNT(1) > 0) clause' */
  found = stmt_preproc.find(L"HAVING (COUNT(1) > 0)", 0);
  if (found != std::wstring::npos) {
    stmt_preproc.erase(found, 21);
  }

  query[L"query"] = toJson(&stmt_preproc);
  auto value = this->doRequest(web::http::methods::POST, L"namespaces/" + this->namespace_ + L"/data/explore/queries", &query);
  return value.at(L"handle").as_string();
}
<|endoftext|>


<commit_before>/* Copyright 2021, The TensorFlow Federated Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License
==============================================================================*/

#include "tensorflow_federated/cc/core/impl/executors/executor_service.h"

#include <memory>
#include <tuple>

#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/match.h"
#include "absl/strings/string_view.h"
#include "absl/synchronization/mutex.h"
#include "tensorflow_federated/cc/core/impl/executors/status_macros.h"
#include "tensorflow_federated/proto/v0/executor.proto.h"

namespace tensorflow_federated {

absl::StatusOr<ValueId> ParseRef(const v0::ValueRef& value_ref_pb) {
  ValueId value_id;
  if (absl::SimpleAtoi(value_ref_pb.id(), &value_id)) {
    return value_id;
  }
  return absl::InvalidArgumentError(absl::StrCat(
      "Could not parse ValueRef from string. Incoming id:", value_ref_pb.id()));
}

absl::StatusOr<std::pair<std::shared_ptr<Executor>, int>>
ExecutorService::RequireExecutor_(std::string method_name) {
  absl::ReaderMutexLock reader_lock(&executor_mutex_);
  if (executor_and_generation_.first == nullptr) {
    return grpc::Status(
        grpc::StatusCode::UNAVAILABLE,
        absl::StrCat("Attempted to call ExecutorService::", method_name,
                     " before setting cardinalities."));
  }
  return executor_and_generation_;
}

ExecutorService::RemoteValueId ExecutorService::CreateRemoteValue_(
    ValueId embedded_value_id, int executor_generation) {
  return absl::StrCat(embedded_value_id, "-", executor_generation);
}

absl::Status MalformattedValueStatus(absl::string_view bad_id) {
  return absl::InvalidArgumentError(
      absl::StrCat("Remote value ID ", bad_id,
                   " malformed: expected to be of the form a-b, where a and "
                   "b are both ints."));
}

absl::Status ExecutorService::EnsureGeneration_(int reference_generation,
                                                int expected_generation) {
  if (reference_generation != expected_generation) {
    return absl::InvalidArgumentError(absl::StrCat(
        "Remote value refers to a non-live executor generation. "
        "Current generation is: ",
        expected_generation,
        "remote valuerefers to generation: ", reference_generation));
  }
  return absl::OkStatus();
}

absl::StatusOr<ValueId> ExecutorService::ResolveRemoteValue_(
    const v0::ValueRef& remote_value_ref, int expected_generation) {
  // Incoming ref should have ID of the form a-b, where a is a
  // uint64 and b s an int. a represents the ValueId in the
  // service's executor, b represents the generation of this executor.
  std::vector<absl::string_view> id_and_generation =
      absl::StrSplit(remote_value_ref.id(), '-');
  if (id_and_generation.size() != 2) {
    return MalformattedValueStatus(remote_value_ref.id());
  }
  int reference_generation;
  if (!absl::SimpleAtoi(id_and_generation[1], &reference_generation)) {
    return MalformattedValueStatus(remote_value_ref.id());
  }
  TFF_TRY(EnsureGeneration_(reference_generation, expected_generation));

  ValueId embedded_value_id;
  if (!absl::SimpleAtoi(id_and_generation[0], &embedded_value_id)) {
    return MalformattedValueStatus(remote_value_ref.id());
  }
  return embedded_value_id;
}

grpc::Status ExecutorService::CreateValue(grpc::ServerContext* context,
                                          const v0::CreateValueRequest* request,
                                          v0::CreateValueResponse* response) {
  std::pair<std::shared_ptr<Executor>, int> executor_and_generation;
  auto [live_executor, used_executor_generation] =
      TFF_TRY(RequireExecutor_("CreateValue"));
  OwnedValueId owned_value_id =
      TFF_TRY(live_executor->CreateValue(request->value()));
  response->mutable_value_ref()->mutable_id()->assign(
      CreateRemoteValue_(owned_value_id.ref(), used_executor_generation));
  // We must call forget on the embedded id to prevent the destructor from
  // running when the variable goes out of scope. Similar considerations apply
  // to the reset of the Create methods below.
  owned_value_id.forget();
  return grpc::Status::OK;
}

grpc::Status ExecutorService::CreateCall(grpc::ServerContext* context,
                                         const v0::CreateCallRequest* request,
                                         v0::CreateCallResponse* response) {
  std::pair<std::shared_ptr<Executor>, int> executor_and_generation;
  auto [live_executor, used_executor_generation] =
      TFF_TRY(RequireExecutor_("CreateCall"));
  ValueId embedded_fn = TFF_TRY(
      ResolveRemoteValue_(request->function_ref(), used_executor_generation));
  std::optional<ValueId> embedded_arg;
  if (request->has_argument_ref()) {
    // Callers should avoid setting the argument ref for invocation of a no-arg
    // fn.
    embedded_arg = TFF_TRY(
        ResolveRemoteValue_(request->argument_ref(), used_executor_generation));
  }
  OwnedValueId called_fn =
      TFF_TRY(live_executor->CreateCall(embedded_fn, embedded_arg));
  response->mutable_value_ref()->mutable_id()->assign(
      CreateRemoteValue_(called_fn.ref(), used_executor_generation));
  // We must prevent this destructor from running similarly to CreateValue.
  called_fn.forget();
  return grpc::Status::OK;
}

grpc::Status ExecutorService::CreateStruct(
    grpc::ServerContext* context, const v0::CreateStructRequest* request,
    v0::CreateStructResponse* response) {
  auto [live_executor, used_executor_generation] =
      TFF_TRY(RequireExecutor_("CreateStruct"));
  std::vector<ValueId> requested_ids;
  requested_ids.reserve(request->element().size());
  for (const v0::CreateStructRequest::Element& elem : request->element()) {
    requested_ids.emplace_back(TFF_TRY(
        ResolveRemoteValue_(elem.value_ref(), used_executor_generation)));
  }
  OwnedValueId created_struct =
      TFF_TRY(live_executor->CreateStruct(requested_ids));
  response->mutable_value_ref()->mutable_id()->assign(
      CreateRemoteValue_(created_struct.ref(), used_executor_generation));
  // We must prevent this destructor from running similarly to CreateValue.
  created_struct.forget();
  return grpc::Status::OK;
}

grpc::Status ExecutorService::CreateSelection(
    grpc::ServerContext* context, const v0::CreateSelectionRequest* request,
    v0::CreateSelectionResponse* response) {
  auto [live_executor, used_executor_generation] =
      TFF_TRY(RequireExecutor_("CreateSelection"));
  ValueId selection_source = TFF_TRY(
      ResolveRemoteValue_(request->source_ref(), used_executor_generation));
  OwnedValueId selected_element = TFF_TRY(
      live_executor->CreateSelection(selection_source, request->index()));
  response->mutable_value_ref()->mutable_id()->assign(
      CreateRemoteValue_(selected_element.ref(), used_executor_generation));
  // We must prevent this destructor from running similarly to CreateValue.
  selected_element.forget();
  return grpc::Status::OK;
}

grpc::Status ExecutorService::Compute(grpc::ServerContext* context,
                                      const v0::ComputeRequest* request,
                                      v0::ComputeResponse* response) {
  auto [live_executor, used_executor_generation] =
      TFF_TRY(RequireExecutor_("Compute"));
  ValueId requested_value = TFF_TRY(
      ResolveRemoteValue_(request->value_ref(), used_executor_generation));
  absl::Status materialize_status =
      live_executor->Materialize(requested_value, response->mutable_value());
  if (!materialize_status.ok()) {
    return materialize_status;
  }
  return grpc::Status::OK;
}

grpc::Status ExecutorService::SetCardinalities(
    grpc::ServerContext* context, const v0::SetCardinalitiesRequest* request,
    v0::SetCardinalitiesResponse* response) {
  CardinalityMap cardinality_map;
  for (const auto& cardinality : request->cardinalities()) {
    cardinality_map.insert(
        {cardinality.placement().uri(), cardinality.cardinality()});
  }
  {
    absl::WriterMutexLock writer_lock(&executor_mutex_);
    auto new_executor = TFF_TRY(executor_factory_(cardinality_map));
    int new_generation = executor_and_generation_.second + 1;
    executor_and_generation_ =
        std::make_pair(std::move(new_executor), new_generation);
  }
  return grpc::Status::OK;
}

grpc::Status ExecutorService::ClearExecutor(
    grpc::ServerContext* context, const v0::ClearExecutorRequest* request,
    v0::ClearExecutorResponse* response) {
  {
    absl::WriterMutexLock writer_lock(&executor_mutex_);
    // Clearing executor should reset executor to null, but without changing the
    // executor's generation; this will be incremented by SetCardinalities.
    executor_and_generation_ =
        std::make_pair(nullptr, executor_and_generation_.second);
  }
  return grpc::Status::OK;
}

grpc::Status ExecutorService::Dispose(grpc::ServerContext* context,
                                      const v0::DisposeRequest* request,
                                      v0::DisposeResponse* response) {
  auto [live_executor, used_executor_generation] =
      TFF_TRY(RequireExecutor_("Dispose"));
  std::vector<ValueId> embedded_ids_to_dispose;
  embedded_ids_to_dispose.reserve(request->value_ref().size());
  // Filter the requested IDs to those corresponding to the currently live
  // executors. Clients are free to batch these requests however they want or
  // let them be called by a GC mechanism, so we do not force the client to only
  // dispose of values in the live executor.
  for (const v0::ValueRef& disposed_value_ref : request->value_ref()) {
    absl::StatusOr<ValueId> embedded_value =
        ResolveRemoteValue_(disposed_value_ref, used_executor_generation);
    if (embedded_value.ok()) {
      TFF_TRY(live_executor->Dispose(embedded_value.ValueOrDie()));
    }
  }
  return grpc::Status::OK;
}

}  // namespace tensorflow_federated
<commit_msg>Use TFF_TRY on final materialize in ExecutorService.<commit_after>/* Copyright 2021, The TensorFlow Federated Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License
==============================================================================*/

#include "tensorflow_federated/cc/core/impl/executors/executor_service.h"

#include <memory>
#include <tuple>

#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/match.h"
#include "absl/strings/string_view.h"
#include "absl/synchronization/mutex.h"
#include "tensorflow_federated/cc/core/impl/executors/status_macros.h"
#include "tensorflow_federated/proto/v0/executor.proto.h"

namespace tensorflow_federated {

absl::StatusOr<ValueId> ParseRef(const v0::ValueRef& value_ref_pb) {
  ValueId value_id;
  if (absl::SimpleAtoi(value_ref_pb.id(), &value_id)) {
    return value_id;
  }
  return absl::InvalidArgumentError(absl::StrCat(
      "Could not parse ValueRef from string. Incoming id:", value_ref_pb.id()));
}

absl::StatusOr<std::pair<std::shared_ptr<Executor>, int>>
ExecutorService::RequireExecutor_(std::string method_name) {
  absl::ReaderMutexLock reader_lock(&executor_mutex_);
  if (executor_and_generation_.first == nullptr) {
    return grpc::Status(
        grpc::StatusCode::UNAVAILABLE,
        absl::StrCat("Attempted to call ExecutorService::", method_name,
                     " before setting cardinalities."));
  }
  return executor_and_generation_;
}

ExecutorService::RemoteValueId ExecutorService::CreateRemoteValue_(
    ValueId embedded_value_id, int executor_generation) {
  return absl::StrCat(embedded_value_id, "-", executor_generation);
}

absl::Status MalformattedValueStatus(absl::string_view bad_id) {
  return absl::InvalidArgumentError(
      absl::StrCat("Remote value ID ", bad_id,
                   " malformed: expected to be of the form a-b, where a and "
                   "b are both ints."));
}

absl::Status ExecutorService::EnsureGeneration_(int reference_generation,
                                                int expected_generation) {
  if (reference_generation != expected_generation) {
    return absl::InvalidArgumentError(absl::StrCat(
        "Remote value refers to a non-live executor generation. "
        "Current generation is: ",
        expected_generation,
        "remote valuerefers to generation: ", reference_generation));
  }
  return absl::OkStatus();
}

absl::StatusOr<ValueId> ExecutorService::ResolveRemoteValue_(
    const v0::ValueRef& remote_value_ref, int expected_generation) {
  // Incoming ref should have ID of the form a-b, where a is a
  // uint64 and b s an int. a represents the ValueId in the
  // service's executor, b represents the generation of this executor.
  std::vector<absl::string_view> id_and_generation =
      absl::StrSplit(remote_value_ref.id(), '-');
  if (id_and_generation.size() != 2) {
    return MalformattedValueStatus(remote_value_ref.id());
  }
  int reference_generation;
  if (!absl::SimpleAtoi(id_and_generation[1], &reference_generation)) {
    return MalformattedValueStatus(remote_value_ref.id());
  }
  TFF_TRY(EnsureGeneration_(reference_generation, expected_generation));

  ValueId embedded_value_id;
  if (!absl::SimpleAtoi(id_and_generation[0], &embedded_value_id)) {
    return MalformattedValueStatus(remote_value_ref.id());
  }
  return embedded_value_id;
}

grpc::Status ExecutorService::CreateValue(grpc::ServerContext* context,
                                          const v0::CreateValueRequest* request,
                                          v0::CreateValueResponse* response) {
  std::pair<std::shared_ptr<Executor>, int> executor_and_generation;
  auto [live_executor, used_executor_generation] =
      TFF_TRY(RequireExecutor_("CreateValue"));
  OwnedValueId owned_value_id =
      TFF_TRY(live_executor->CreateValue(request->value()));
  response->mutable_value_ref()->mutable_id()->assign(
      CreateRemoteValue_(owned_value_id.ref(), used_executor_generation));
  // We must call forget on the embedded id to prevent the destructor from
  // running when the variable goes out of scope. Similar considerations apply
  // to the reset of the Create methods below.
  owned_value_id.forget();
  return grpc::Status::OK;
}

grpc::Status ExecutorService::CreateCall(grpc::ServerContext* context,
                                         const v0::CreateCallRequest* request,
                                         v0::CreateCallResponse* response) {
  std::pair<std::shared_ptr<Executor>, int> executor_and_generation;
  auto [live_executor, used_executor_generation] =
      TFF_TRY(RequireExecutor_("CreateCall"));
  ValueId embedded_fn = TFF_TRY(
      ResolveRemoteValue_(request->function_ref(), used_executor_generation));
  std::optional<ValueId> embedded_arg;
  if (request->has_argument_ref()) {
    // Callers should avoid setting the argument ref for invocation of a no-arg
    // fn.
    embedded_arg = TFF_TRY(
        ResolveRemoteValue_(request->argument_ref(), used_executor_generation));
  }
  OwnedValueId called_fn =
      TFF_TRY(live_executor->CreateCall(embedded_fn, embedded_arg));
  response->mutable_value_ref()->mutable_id()->assign(
      CreateRemoteValue_(called_fn.ref(), used_executor_generation));
  // We must prevent this destructor from running similarly to CreateValue.
  called_fn.forget();
  return grpc::Status::OK;
}

grpc::Status ExecutorService::CreateStruct(
    grpc::ServerContext* context, const v0::CreateStructRequest* request,
    v0::CreateStructResponse* response) {
  auto [live_executor, used_executor_generation] =
      TFF_TRY(RequireExecutor_("CreateStruct"));
  std::vector<ValueId> requested_ids;
  requested_ids.reserve(request->element().size());
  for (const v0::CreateStructRequest::Element& elem : request->element()) {
    requested_ids.emplace_back(TFF_TRY(
        ResolveRemoteValue_(elem.value_ref(), used_executor_generation)));
  }
  OwnedValueId created_struct =
      TFF_TRY(live_executor->CreateStruct(requested_ids));
  response->mutable_value_ref()->mutable_id()->assign(
      CreateRemoteValue_(created_struct.ref(), used_executor_generation));
  // We must prevent this destructor from running similarly to CreateValue.
  created_struct.forget();
  return grpc::Status::OK;
}

grpc::Status ExecutorService::CreateSelection(
    grpc::ServerContext* context, const v0::CreateSelectionRequest* request,
    v0::CreateSelectionResponse* response) {
  auto [live_executor, used_executor_generation] =
      TFF_TRY(RequireExecutor_("CreateSelection"));
  ValueId selection_source = TFF_TRY(
      ResolveRemoteValue_(request->source_ref(), used_executor_generation));
  OwnedValueId selected_element = TFF_TRY(
      live_executor->CreateSelection(selection_source, request->index()));
  response->mutable_value_ref()->mutable_id()->assign(
      CreateRemoteValue_(selected_element.ref(), used_executor_generation));
  // We must prevent this destructor from running similarly to CreateValue.
  selected_element.forget();
  return grpc::Status::OK;
}

grpc::Status ExecutorService::Compute(grpc::ServerContext* context,
                                      const v0::ComputeRequest* request,
                                      v0::ComputeResponse* response) {
  auto [live_executor, used_executor_generation] =
      TFF_TRY(RequireExecutor_("Compute"));
  ValueId requested_value = TFF_TRY(
      ResolveRemoteValue_(request->value_ref(), used_executor_generation));
  TFF_TRY(
      live_executor->Materialize(requested_value, response->mutable_value()));
  return grpc::Status::OK;
}

grpc::Status ExecutorService::SetCardinalities(
    grpc::ServerContext* context, const v0::SetCardinalitiesRequest* request,
    v0::SetCardinalitiesResponse* response) {
  CardinalityMap cardinality_map;
  for (const auto& cardinality : request->cardinalities()) {
    cardinality_map.insert(
        {cardinality.placement().uri(), cardinality.cardinality()});
  }
  {
    absl::WriterMutexLock writer_lock(&executor_mutex_);
    auto new_executor = TFF_TRY(executor_factory_(cardinality_map));
    int new_generation = executor_and_generation_.second + 1;
    executor_and_generation_ =
        std::make_pair(std::move(new_executor), new_generation);
  }
  return grpc::Status::OK;
}

grpc::Status ExecutorService::ClearExecutor(
    grpc::ServerContext* context, const v0::ClearExecutorRequest* request,
    v0::ClearExecutorResponse* response) {
  {
    absl::WriterMutexLock writer_lock(&executor_mutex_);
    // Clearing executor should reset executor to null, but without changing the
    // executor's generation; this will be incremented by SetCardinalities.
    executor_and_generation_ =
        std::make_pair(nullptr, executor_and_generation_.second);
  }
  return grpc::Status::OK;
}

grpc::Status ExecutorService::Dispose(grpc::ServerContext* context,
                                      const v0::DisposeRequest* request,
                                      v0::DisposeResponse* response) {
  auto [live_executor, used_executor_generation] =
      TFF_TRY(RequireExecutor_("Dispose"));
  std::vector<ValueId> embedded_ids_to_dispose;
  embedded_ids_to_dispose.reserve(request->value_ref().size());
  // Filter the requested IDs to those corresponding to the currently live
  // executors. Clients are free to batch these requests however they want or
  // let them be called by a GC mechanism, so we do not force the client to only
  // dispose of values in the live executor.
  for (const v0::ValueRef& disposed_value_ref : request->value_ref()) {
    absl::StatusOr<ValueId> embedded_value =
        ResolveRemoteValue_(disposed_value_ref, used_executor_generation);
    if (embedded_value.ok()) {
      TFF_TRY(live_executor->Dispose(embedded_value.ValueOrDie()));
    }
  }
  return grpc::Status::OK;
}

}  // namespace tensorflow_federated
<|endoftext|>


<commit_before>/*
Resembla: Word-based Japanese similar sentence search library
https://github.com/tuem/resembla

Copyright 2017 Takashi Uemura

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#ifndef RESEMBLA_TEXT_CLASSIFICATION_FEATURE_EXTRACTOR_HPP
#define RESEMBLA_TEXT_CLASSIFICATION_FEATURE_EXTRACTOR_HPP

#include <string>
#include <unordered_map>
#include <mutex>

#include <mecab.h>
#include <libsvm/svm.h>

#include "feature_extractor.hpp"

namespace resembla {

struct TextClassificationFeatureExtractor: public FeatureExtractor::Function
{
    TextClassificationFeatureExtractor(const std::string& mecab_options,
            const std::string& dict_path, const std::string& model_path);

    Feature::text_type operator()(const string_type& text) const;

protected:
    using Dictionary = std::unordered_map<std::string, int>;
    Dictionary dictionary;

    std::shared_ptr<MeCab::Tagger> tagger;
    mutable std::mutex mutex_tagger;

    svm_model *model;
    mutable std::mutex mutex_model;

    std::vector<svm_node> toNodes(const string_type& x) const;
};

}
#endif
<commit_msg>delete type definition<commit_after>/*
Resembla: Word-based Japanese similar sentence search library
https://github.com/tuem/resembla

Copyright 2017 Takashi Uemura

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#ifndef RESEMBLA_TEXT_CLASSIFICATION_FEATURE_EXTRACTOR_HPP
#define RESEMBLA_TEXT_CLASSIFICATION_FEATURE_EXTRACTOR_HPP

#include <string>
#include <unordered_map>
#include <mutex>

#include <mecab.h>
#include <libsvm/svm.h>

#include "feature_extractor.hpp"

namespace resembla {

struct TextClassificationFeatureExtractor: public FeatureExtractor::Function
{
    TextClassificationFeatureExtractor(const std::string& mecab_options,
            const std::string& dict_path, const std::string& model_path);

    Feature::text_type operator()(const string_type& text) const;

protected:
    std::unordered_map<std::string, int> dictionary;

    std::shared_ptr<MeCab::Tagger> tagger;
    mutable std::mutex mutex_tagger;

    svm_model *model;
    mutable std::mutex mutex_model;

    std::vector<svm_node> toNodes(const string_type& x) const;
};

}
#endif
<|endoftext|>


<commit_before>//===----------------- lib/Jit/options.cpp ----------------------*- C++ -*-===//
//
// LLILC
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license.
// See LICENSE file in the project root for full license information.
//
//===----------------------------------------------------------------------===//
///
/// \file
/// \brief Definition of the Options class that encapsulates JIT options
///        extracted from CoreCLR config values.
///
//===----------------------------------------------------------------------===//

#include "earlyincludes.h"
#include "global.h"
#include "jitpch.h"
#include "LLILCJit.h"
#include "jitoptions.h"

// Define a macro for cross-platform UTF-16 string literals.
#if defined(_MSC_VER)
#define UTF16(lit) L##lit
#else
#define UTF16(lit) u##lit
#endif

// For now we're always running as the altjit
#define ALT_JIT 1

// These are the instantiations of the static method sets in Options.h.
// This MethodSets are initialized from CLRConfig values passed through
// the corinfo.h interface.

MethodSet JitOptions::AltJitMethodSet;
MethodSet JitOptions::AltJitNgenMethodSet;
MethodSet JitOptions::ExcludeMethodSet;
MethodSet JitOptions::BreakMethodSet;
MethodSet JitOptions::MSILMethodSet;
MethodSet JitOptions::LLVMMethodSet;
MethodSet JitOptions::CodeRangeMethodSet;

template <typename UTF16CharT>
char16_t *getStringConfigValue(ICorJitInfo *CorInfo, const UTF16CharT *Name) {
  static_assert(sizeof(UTF16CharT) == 2, "UTF16CharT is the wrong size!");
  return (char16_t *)CorInfo->getStringConfigValue((const wchar_t *)Name);
}

template <typename UTF16CharT>
void freeStringConfigValue(ICorJitInfo *CorInfo, UTF16CharT *Value) {
  static_assert(sizeof(UTF16CharT) == 2, "UTF16CharT is the wrong size!");
  return CorInfo->freeStringConfigValue((wchar_t *)Value);
}

JitOptions::JitOptions(LLILCJitContext &Context) {
  // Set 'IsAltJit' based on environment information.
  IsAltJit = queryIsAltJit(Context);

  // Set dump level for this JIT invocation.
  DumpLevel = queryDumpLevel(Context);

  // Set optimization level for this JIT invocation.
  OptLevel = queryOptLevel(Context);
  EnableOptimization = OptLevel != OptLevel::DEBUG_CODE;

  // Set whether to use conservative GC.
  UseConservativeGC = queryUseConservativeGC(Context);

  // Set whether to insert statepoints.
  DoInsertStatepoints = queryDoInsertStatepoints(Context);

  DoSIMDIntrinsic = queryDoSIMDIntrinsic(Context);

  // Set whether to do tail call opt.
  DoTailCallOpt = queryDoTailCallOpt(Context);

  LogGcInfo = queryLogGcInfo(Context);

  // Set whether to insert failfast in exception handlers.
  ExecuteHandlers = queryExecuteHandlers(Context);

  IsExcludeMethod = queryIsExcludeMethod(Context);
  IsBreakMethod = queryIsBreakMethod(Context);
  IsMSILDumpMethod = queryIsMSILDumpMethod(Context);
  IsLLVMDumpMethod = queryIsLLVMDumpMethod(Context);
  IsCodeRangeMethod = queryIsCodeRangeMethod(Context);

  if (IsAltJit) {
    PreferredIntrinsicSIMDVectorLength = 0;
  } else {
    PreferredIntrinsicSIMDVectorLength = 32;
  }

  // Validate Statepoint and Conservative GC state.
  assert(DoInsertStatepoints ||
         UseConservativeGC && "Statepoints required for precise-GC");
}

bool JitOptions::queryDoTailCallOpt(LLILCJitContext &Context) {
  return (bool)DEFAULT_TAIL_CALL_OPT;
}

bool JitOptions::queryIsAltJit(LLILCJitContext &Context) {
  // Initial state is that we are not an alternative jit until proven otherwise;
  bool IsAlternateJit = false;

// NDEBUG is !Debug

#if !defined(NDEBUG)

  // DEBUG case

  // Get/reuse method set that contains the altjit method value.
  MethodSet *AltJit = nullptr;

  if (Context.Flags & CORJIT_FLG_PREJIT) {
    if (!AltJitNgenMethodSet.isInitialized()) {
      char16_t *NgenStr =
          getStringConfigValue(Context.JitInfo, UTF16("AltJitNgen"));
      std::unique_ptr<std::string> NgenUtf8 = Convert::utf16ToUtf8(NgenStr);
      AltJitNgenMethodSet.init(std::move(NgenUtf8));
      freeStringConfigValue(Context.JitInfo, NgenStr);
    }
    // Set up AltJitNgen set to be used for AltJit test.
    AltJit = &AltJitNgenMethodSet;
  } else {
    if (!AltJitMethodSet.isInitialized()) {
      char16_t *JitStr = getStringConfigValue(Context.JitInfo, UTF16("AltJit"));
      // Move this to the UTIL code and ifdef it for platform
      std::unique_ptr<std::string> JitUtf8 = Convert::utf16ToUtf8(JitStr);
      AltJitMethodSet.init(std::move(JitUtf8));
      freeStringConfigValue(Context.JitInfo, JitStr);
    }
    // Set up AltJit set to be use for AltJit test.
    AltJit = &AltJitMethodSet;
  }

#ifdef ALT_JIT
  const char *ClassName = nullptr;
  const char *MethodName = nullptr;
  MethodName =
      Context.JitInfo->getMethodName(Context.MethodInfo->ftn, &ClassName);
  IsAlternateJit =
      AltJit->contains(MethodName, ClassName, Context.MethodInfo->args.pSig);
#endif // ALT_JIT

#else
  if (Context.Flags & CORJIT_FLG_PREJIT) {
    char16_t *NgenStr =
        getStringConfigValue(Context.JitInfo, UTF16("AltJitNgen"));
    std::unique_ptr<std::string> NgenUtf8 = Convert::utf16ToUtf8(NgenStr);
    if (NgenUtf8->compare("*") == 0) {
      IsAlternateJit = true;
    }
  } else {
    char16_t *JitStr = getStringConfigValue(Context.JitInfo, UTF16("AltJit"));
    std::unique_ptr<std::string> JitUtf8 = Convert::utf16ToUtf8(JitStr);
    if (JitUtf8->compare("*") == 0) {
      IsAlternateJit = true;
    }
  }
#endif

  return IsAlternateJit;
}

::DumpLevel JitOptions::queryDumpLevel(LLILCJitContext &Context) {
  ::DumpLevel JitDumpLevel = ::DumpLevel::NODUMP;

  char16_t *LevelWStr =
      getStringConfigValue(Context.JitInfo, UTF16("DUMPLLVMIR"));
  if (LevelWStr) {
    std::unique_ptr<std::string> Level = Convert::utf16ToUtf8(LevelWStr);
    std::transform(Level->begin(), Level->end(), Level->begin(), ::toupper);
    if (Level->compare("VERBOSE") == 0) {
      JitDumpLevel = ::DumpLevel::VERBOSE;
    } else if (Level->compare("SUMMARY") == 0) {
      JitDumpLevel = ::DumpLevel::SUMMARY;
    }
  }

  return JitDumpLevel;
}

bool JitOptions::queryIsExcludeMethod(LLILCJitContext &JitContext) {
  return queryMethodSet(JitContext, ExcludeMethodSet,
                        (const char16_t *)UTF16("AltJitExclude"));
}

bool JitOptions::queryIsBreakMethod(LLILCJitContext &JitContext) {
  return queryMethodSet(JitContext, BreakMethodSet,
                        (const char16_t *)UTF16("AltJitBreakAtJitStart"));
}

bool JitOptions::queryIsMSILDumpMethod(LLILCJitContext &JitContext) {
  return queryMethodSet(JitContext, MSILMethodSet,
                        (const char16_t *)UTF16("AltJitMSILDump"));
}

bool JitOptions::queryIsLLVMDumpMethod(LLILCJitContext &JitContext) {
  return queryMethodSet(JitContext, LLVMMethodSet,
                        (const char16_t *)UTF16("AltJitLLVMDump"));
}

bool JitOptions::queryIsCodeRangeMethod(LLILCJitContext &JitContext) {
  return queryMethodSet(JitContext, CodeRangeMethodSet,
                        (const char16_t *)UTF16("AltJitCodeRangeDump"));
}

bool JitOptions::queryMethodSet(LLILCJitContext &JitContext, MethodSet &TheSet,
                                const char16_t *Name) {
  if (!TheSet.isInitialized()) {
    char16_t *ConfigStr = getStringConfigValue(JitContext.JitInfo, Name);
    bool NeedFree = true;
    if (ConfigStr == nullptr) {
      ConfigStr = const_cast<char16_t *>((const char16_t *)UTF16(""));
      NeedFree = false;
    }
    std::unique_ptr<std::string> ConfigUtf8 = Convert::utf16ToUtf8(ConfigStr);
    TheSet.init(std::move(ConfigUtf8));
    if (NeedFree) {
      freeStringConfigValue(JitContext.JitInfo, ConfigStr);
    }
  }

  const char *ClassName = nullptr;
  const char *MethodName = nullptr;
  MethodName =
      JitContext.JitInfo->getMethodName(JitContext.MethodInfo->ftn, &ClassName);
  bool IsInMethodSet =
      TheSet.contains(MethodName, ClassName, JitContext.MethodInfo->args.pSig);
  return IsInMethodSet;
}

bool JitOptions::queryNonNullNonEmpty(LLILCJitContext &JitContext,
                                      const char16_t *Name) {
  char16_t *ConfigStr = getStringConfigValue(JitContext.JitInfo, Name);
  return (ConfigStr != nullptr) && (*ConfigStr != 0);
}

// Get the GC-Scheme used by the runtime -- conservative/precise
bool JitOptions::queryUseConservativeGC(LLILCJitContext &Context) {
  return queryNonNullNonEmpty(Context,
                              (const char16_t *)UTF16("gcConservative"));
}

// Determine if GC statepoints should be inserted.
bool JitOptions::queryDoInsertStatepoints(LLILCJitContext &Context) {
  return queryNonNullNonEmpty(Context,
                              (const char16_t *)UTF16("INSERTSTATEPOINTS"));
}

// Determine if GCInfo encoding logs should be emitted
bool JitOptions::queryLogGcInfo(LLILCJitContext &Context) {
  return queryNonNullNonEmpty(Context,
                              (const char16_t *)UTF16("JitGCInfoLogging"));
}

// Determine if exception handlers should be executed.
bool JitOptions::queryExecuteHandlers(LLILCJitContext &Context) {
  return queryNonNullNonEmpty(Context,
                              (const char16_t *)UTF16("ExecuteHandlers"));
}

// Determine if SIMD intrinsics should be used.
bool JitOptions::queryDoSIMDIntrinsic(LLILCJitContext &Context) {
  return queryNonNullNonEmpty(Context,
                              (const char16_t *)UTF16("SIMDINTRINSIC"));
}

OptLevel JitOptions::queryOptLevel(LLILCJitContext &Context) {
  ::OptLevel JitOptLevel = ::OptLevel::BLENDED_CODE;
  // Currently we only check for the debug flag but this will be extended
  // to account for further opt levels as we move forward.
  if ((Context.Flags & CORJIT_FLG_DEBUG_CODE) != 0) {
    JitOptLevel = ::OptLevel::DEBUG_CODE;
  }

  return JitOptLevel;
}

JitOptions::~JitOptions() {}
<commit_msg>Fix build issue on NetBSD: Correct namespace of OptLevel::DEBUG_CODE<commit_after>//===----------------- lib/Jit/options.cpp ----------------------*- C++ -*-===//
//
// LLILC
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license.
// See LICENSE file in the project root for full license information.
//
//===----------------------------------------------------------------------===//
///
/// \file
/// \brief Definition of the Options class that encapsulates JIT options
///        extracted from CoreCLR config values.
///
//===----------------------------------------------------------------------===//

#include "earlyincludes.h"
#include "global.h"
#include "jitpch.h"
#include "LLILCJit.h"
#include "jitoptions.h"

// Define a macro for cross-platform UTF-16 string literals.
#if defined(_MSC_VER)
#define UTF16(lit) L##lit
#else
#define UTF16(lit) u##lit
#endif

// For now we're always running as the altjit
#define ALT_JIT 1

// These are the instantiations of the static method sets in Options.h.
// This MethodSets are initialized from CLRConfig values passed through
// the corinfo.h interface.

MethodSet JitOptions::AltJitMethodSet;
MethodSet JitOptions::AltJitNgenMethodSet;
MethodSet JitOptions::ExcludeMethodSet;
MethodSet JitOptions::BreakMethodSet;
MethodSet JitOptions::MSILMethodSet;
MethodSet JitOptions::LLVMMethodSet;
MethodSet JitOptions::CodeRangeMethodSet;

template <typename UTF16CharT>
char16_t *getStringConfigValue(ICorJitInfo *CorInfo, const UTF16CharT *Name) {
  static_assert(sizeof(UTF16CharT) == 2, "UTF16CharT is the wrong size!");
  return (char16_t *)CorInfo->getStringConfigValue((const wchar_t *)Name);
}

template <typename UTF16CharT>
void freeStringConfigValue(ICorJitInfo *CorInfo, UTF16CharT *Value) {
  static_assert(sizeof(UTF16CharT) == 2, "UTF16CharT is the wrong size!");
  return CorInfo->freeStringConfigValue((wchar_t *)Value);
}

JitOptions::JitOptions(LLILCJitContext &Context) {
  // Set 'IsAltJit' based on environment information.
  IsAltJit = queryIsAltJit(Context);

  // Set dump level for this JIT invocation.
  DumpLevel = queryDumpLevel(Context);

  // Set optimization level for this JIT invocation.
  OptLevel = queryOptLevel(Context);
  EnableOptimization = OptLevel != ::OptLevel::DEBUG_CODE;

  // Set whether to use conservative GC.
  UseConservativeGC = queryUseConservativeGC(Context);

  // Set whether to insert statepoints.
  DoInsertStatepoints = queryDoInsertStatepoints(Context);

  DoSIMDIntrinsic = queryDoSIMDIntrinsic(Context);

  // Set whether to do tail call opt.
  DoTailCallOpt = queryDoTailCallOpt(Context);

  LogGcInfo = queryLogGcInfo(Context);

  // Set whether to insert failfast in exception handlers.
  ExecuteHandlers = queryExecuteHandlers(Context);

  IsExcludeMethod = queryIsExcludeMethod(Context);
  IsBreakMethod = queryIsBreakMethod(Context);
  IsMSILDumpMethod = queryIsMSILDumpMethod(Context);
  IsLLVMDumpMethod = queryIsLLVMDumpMethod(Context);
  IsCodeRangeMethod = queryIsCodeRangeMethod(Context);

  if (IsAltJit) {
    PreferredIntrinsicSIMDVectorLength = 0;
  } else {
    PreferredIntrinsicSIMDVectorLength = 32;
  }

  // Validate Statepoint and Conservative GC state.
  assert(DoInsertStatepoints ||
         UseConservativeGC && "Statepoints required for precise-GC");
}

bool JitOptions::queryDoTailCallOpt(LLILCJitContext &Context) {
  return (bool)DEFAULT_TAIL_CALL_OPT;
}

bool JitOptions::queryIsAltJit(LLILCJitContext &Context) {
  // Initial state is that we are not an alternative jit until proven otherwise;
  bool IsAlternateJit = false;

// NDEBUG is !Debug

#if !defined(NDEBUG)

  // DEBUG case

  // Get/reuse method set that contains the altjit method value.
  MethodSet *AltJit = nullptr;

  if (Context.Flags & CORJIT_FLG_PREJIT) {
    if (!AltJitNgenMethodSet.isInitialized()) {
      char16_t *NgenStr =
          getStringConfigValue(Context.JitInfo, UTF16("AltJitNgen"));
      std::unique_ptr<std::string> NgenUtf8 = Convert::utf16ToUtf8(NgenStr);
      AltJitNgenMethodSet.init(std::move(NgenUtf8));
      freeStringConfigValue(Context.JitInfo, NgenStr);
    }
    // Set up AltJitNgen set to be used for AltJit test.
    AltJit = &AltJitNgenMethodSet;
  } else {
    if (!AltJitMethodSet.isInitialized()) {
      char16_t *JitStr = getStringConfigValue(Context.JitInfo, UTF16("AltJit"));
      // Move this to the UTIL code and ifdef it for platform
      std::unique_ptr<std::string> JitUtf8 = Convert::utf16ToUtf8(JitStr);
      AltJitMethodSet.init(std::move(JitUtf8));
      freeStringConfigValue(Context.JitInfo, JitStr);
    }
    // Set up AltJit set to be use for AltJit test.
    AltJit = &AltJitMethodSet;
  }

#ifdef ALT_JIT
  const char *ClassName = nullptr;
  const char *MethodName = nullptr;
  MethodName =
      Context.JitInfo->getMethodName(Context.MethodInfo->ftn, &ClassName);
  IsAlternateJit =
      AltJit->contains(MethodName, ClassName, Context.MethodInfo->args.pSig);
#endif // ALT_JIT

#else
  if (Context.Flags & CORJIT_FLG_PREJIT) {
    char16_t *NgenStr =
        getStringConfigValue(Context.JitInfo, UTF16("AltJitNgen"));
    std::unique_ptr<std::string> NgenUtf8 = Convert::utf16ToUtf8(NgenStr);
    if (NgenUtf8->compare("*") == 0) {
      IsAlternateJit = true;
    }
  } else {
    char16_t *JitStr = getStringConfigValue(Context.JitInfo, UTF16("AltJit"));
    std::unique_ptr<std::string> JitUtf8 = Convert::utf16ToUtf8(JitStr);
    if (JitUtf8->compare("*") == 0) {
      IsAlternateJit = true;
    }
  }
#endif

  return IsAlternateJit;
}

::DumpLevel JitOptions::queryDumpLevel(LLILCJitContext &Context) {
  ::DumpLevel JitDumpLevel = ::DumpLevel::NODUMP;

  char16_t *LevelWStr =
      getStringConfigValue(Context.JitInfo, UTF16("DUMPLLVMIR"));
  if (LevelWStr) {
    std::unique_ptr<std::string> Level = Convert::utf16ToUtf8(LevelWStr);
    std::transform(Level->begin(), Level->end(), Level->begin(), ::toupper);
    if (Level->compare("VERBOSE") == 0) {
      JitDumpLevel = ::DumpLevel::VERBOSE;
    } else if (Level->compare("SUMMARY") == 0) {
      JitDumpLevel = ::DumpLevel::SUMMARY;
    }
  }

  return JitDumpLevel;
}

bool JitOptions::queryIsExcludeMethod(LLILCJitContext &JitContext) {
  return queryMethodSet(JitContext, ExcludeMethodSet,
                        (const char16_t *)UTF16("AltJitExclude"));
}

bool JitOptions::queryIsBreakMethod(LLILCJitContext &JitContext) {
  return queryMethodSet(JitContext, BreakMethodSet,
                        (const char16_t *)UTF16("AltJitBreakAtJitStart"));
}

bool JitOptions::queryIsMSILDumpMethod(LLILCJitContext &JitContext) {
  return queryMethodSet(JitContext, MSILMethodSet,
                        (const char16_t *)UTF16("AltJitMSILDump"));
}

bool JitOptions::queryIsLLVMDumpMethod(LLILCJitContext &JitContext) {
  return queryMethodSet(JitContext, LLVMMethodSet,
                        (const char16_t *)UTF16("AltJitLLVMDump"));
}

bool JitOptions::queryIsCodeRangeMethod(LLILCJitContext &JitContext) {
  return queryMethodSet(JitContext, CodeRangeMethodSet,
                        (const char16_t *)UTF16("AltJitCodeRangeDump"));
}

bool JitOptions::queryMethodSet(LLILCJitContext &JitContext, MethodSet &TheSet,
                                const char16_t *Name) {
  if (!TheSet.isInitialized()) {
    char16_t *ConfigStr = getStringConfigValue(JitContext.JitInfo, Name);
    bool NeedFree = true;
    if (ConfigStr == nullptr) {
      ConfigStr = const_cast<char16_t *>((const char16_t *)UTF16(""));
      NeedFree = false;
    }
    std::unique_ptr<std::string> ConfigUtf8 = Convert::utf16ToUtf8(ConfigStr);
    TheSet.init(std::move(ConfigUtf8));
    if (NeedFree) {
      freeStringConfigValue(JitContext.JitInfo, ConfigStr);
    }
  }

  const char *ClassName = nullptr;
  const char *MethodName = nullptr;
  MethodName =
      JitContext.JitInfo->getMethodName(JitContext.MethodInfo->ftn, &ClassName);
  bool IsInMethodSet =
      TheSet.contains(MethodName, ClassName, JitContext.MethodInfo->args.pSig);
  return IsInMethodSet;
}

bool JitOptions::queryNonNullNonEmpty(LLILCJitContext &JitContext,
                                      const char16_t *Name) {
  char16_t *ConfigStr = getStringConfigValue(JitContext.JitInfo, Name);
  return (ConfigStr != nullptr) && (*ConfigStr != 0);
}

// Get the GC-Scheme used by the runtime -- conservative/precise
bool JitOptions::queryUseConservativeGC(LLILCJitContext &Context) {
  return queryNonNullNonEmpty(Context,
                              (const char16_t *)UTF16("gcConservative"));
}

// Determine if GC statepoints should be inserted.
bool JitOptions::queryDoInsertStatepoints(LLILCJitContext &Context) {
  return queryNonNullNonEmpty(Context,
                              (const char16_t *)UTF16("INSERTSTATEPOINTS"));
}

// Determine if GCInfo encoding logs should be emitted
bool JitOptions::queryLogGcInfo(LLILCJitContext &Context) {
  return queryNonNullNonEmpty(Context,
                              (const char16_t *)UTF16("JitGCInfoLogging"));
}

// Determine if exception handlers should be executed.
bool JitOptions::queryExecuteHandlers(LLILCJitContext &Context) {
  return queryNonNullNonEmpty(Context,
                              (const char16_t *)UTF16("ExecuteHandlers"));
}

// Determine if SIMD intrinsics should be used.
bool JitOptions::queryDoSIMDIntrinsic(LLILCJitContext &Context) {
  return queryNonNullNonEmpty(Context,
                              (const char16_t *)UTF16("SIMDINTRINSIC"));
}

OptLevel JitOptions::queryOptLevel(LLILCJitContext &Context) {
  ::OptLevel JitOptLevel = ::OptLevel::BLENDED_CODE;
  // Currently we only check for the debug flag but this will be extended
  // to account for further opt levels as we move forward.
  if ((Context.Flags & CORJIT_FLG_DEBUG_CODE) != 0) {
    JitOptLevel = ::OptLevel::DEBUG_CODE;
  }

  return JitOptLevel;
}

JitOptions::~JitOptions() {}
<|endoftext|>


<commit_before><commit_msg>linux: raise logging level since INFO doesn't show in Release (?)<commit_after><|endoftext|>


<commit_before><commit_msg>Fix build.<commit_after><|endoftext|>


<commit_before>
//
// This source file is part of appleseed.
// Visit http://appleseedhq.net/ for additional information and resources.
//
// This software is released under the MIT license.
//
// Copyright (c) 2010-2012 Francois Beaune, Jupiter Jazz
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//

// Interface header.
#include "localaccumulationframebuffer.h"

// appleseed.renderer headers.
#include "renderer/kernel/rendering/sample.h"
#include "renderer/modeling/frame/frame.h"

// appleseed.foundation headers.
#include "foundation/image/canvasproperties.h"
#include "foundation/image/image.h"
#include "foundation/image/pixel.h"
#include "foundation/math/scalar.h"
#include "foundation/platform/compiler.h"
#include "foundation/platform/thread.h"

// Standard headers.
#include <cassert>
#include <algorithm>

using namespace boost;
using namespace foundation;
using namespace std;

namespace renderer
{

// If defined, draw each pixel with a shade of gray proportional to the number of samples it contains.
#undef DEBUG_DISPLAY_SAMPLE_COUNT

LocalAccumulationFramebuffer::LocalAccumulationFramebuffer(
    const size_t    width,
    const size_t    height)
  : AccumulationFramebuffer(width, height)
{
    // todo: change to static_assert<>.
    assert(sizeof(AccumulationPixel) == 5 * sizeof(float));

    const size_t MinSize = 32;

    size_t level_width = width;
    size_t level_height = height;

    do
    {
        m_levels.push_back(
            new Tile(
                max(level_width, MinSize),
                max(level_height, MinSize),
                4 + 1,
                PixelFormatFloat));

        m_set_pixels.push_back(0);

        level_width /= 2;
        level_height /= 2;
    }
    while (level_width > MinSize && level_height > MinSize);

    clear();
}

LocalAccumulationFramebuffer::~LocalAccumulationFramebuffer()
{
    for (size_t i = 0; i < m_levels.size(); ++i)
        delete m_levels[i];
}

void LocalAccumulationFramebuffer::clear()
{
    Spinlock::ScopedLock lock(m_spinlock);

    AccumulationFramebuffer::clear_no_lock();

    for (size_t level = 0; level < m_levels.size(); ++level)
    {
        Tile* tile = m_levels[level];

        AccumulationPixel* pixel = reinterpret_cast<AccumulationPixel*>(tile->pixel(0));
        const size_t pixel_count = tile->get_pixel_count();

        for (size_t i = 0; i < pixel_count; ++i)
        {
            pixel[i].m_color.set(0.0f);
            pixel[i].m_count = 0;
        }

        m_set_pixels[level] = 0;
    }

    m_current_level = m_levels.size() - 1;
}

void LocalAccumulationFramebuffer::store_samples(
    const size_t    sample_count,
    const Sample    samples[])
{
    Spinlock::ScopedLock lock(m_spinlock);

    const Sample* RESTRICT sample_ptr = samples;
    const Sample* RESTRICT sample_end = samples + sample_count;

    if (m_current_level == 0)
    {
        Tile* tile = m_levels[0];

        const double fb_width = static_cast<double>(tile->get_width());
        const double fb_height = static_cast<double>(tile->get_height());

        while (sample_ptr < sample_end)
        {
            const double fx = sample_ptr->m_position.x * fb_width;
            const double fy = sample_ptr->m_position.y * fb_height;
            const size_t x = truncate<size_t>(fx);
            const size_t y = truncate<size_t>(fy);

            AccumulationPixel* pixel =
                reinterpret_cast<AccumulationPixel*>(tile->pixel(x, y));

            pixel->m_color += sample_ptr->m_color;
            pixel->m_count += 1;

            if (pixel->m_count == 1)
                ++m_set_pixels[0];

            ++sample_ptr;
        }
    }
    else
    {
        while (sample_ptr < sample_end)
        {
            for (size_t level = 0; level <= m_current_level; ++level)
            {
                Tile* tile = m_levels[level];

                const double fx = sample_ptr->m_position.x * tile->get_width();
                const double fy = sample_ptr->m_position.y * tile->get_height();
                const size_t x = truncate<size_t>(fx);
                const size_t y = truncate<size_t>(fy);

                AccumulationPixel* pixel =
                    reinterpret_cast<AccumulationPixel*>(tile->pixel(x, y));

                pixel->m_color += sample_ptr->m_color;
                pixel->m_count += 1;

                if (pixel->m_count == 1)
                {
                    if (++m_set_pixels[level] == tile->get_pixel_count())
                    {
                        --m_current_level;
                        break;
                    }
                }
            }

            ++sample_ptr;
        }
    }

    m_sample_count += sample_count;
}

void LocalAccumulationFramebuffer::develop_to_frame(Frame& frame) const
{
    Image& image = frame.image();
    const CanvasProperties& frame_props = image.properties();

    assert(frame_props.m_canvas_width == m_width);
    assert(frame_props.m_canvas_height == m_height);
    assert(frame_props.m_channel_count == 4);

    for (size_t ty = 0; ty < frame_props.m_tile_count_y; ++ty)
    {
        for (size_t tx = 0; tx < frame_props.m_tile_count_x; ++tx)
        {
            Tile& tile = image.tile(tx, ty);

            const size_t origin_x = tx * frame_props.m_tile_width;
            const size_t origin_y = ty * frame_props.m_tile_height;

            develop_to_tile(tile, origin_x, origin_y, tx, ty);
        }
    }
}

void LocalAccumulationFramebuffer::develop_to_tile(
    Tile&           tile,
    const size_t    origin_x,
    const size_t    origin_y,
    const size_t    tile_x,
    const size_t    tile_y) const
{
    const size_t display_level =
        m_current_level > 0 || m_set_pixels[0] < m_pixel_count
            ? min(m_current_level + 1, m_levels.size() - 1)
            : 0;

    const Tile* src_tile = m_levels[display_level];

    const size_t tile_width = tile.get_width();
    const size_t tile_height = tile.get_height();

    for (size_t y = 0; y < tile_height; ++y)
    {
        for (size_t x = 0; x < tile_width; ++x)
        {
#ifdef DEBUG_DISPLAY_SAMPLE_COUNT

            const AccumulationPixel* pixel =
                reinterpret_cast<const AccumulationPixel*>(
                    m_tile->pixel(origin_x + x, origin_y + y));

            const float c = saturate(static_cast<float>(pixel->m_count) / 20.0f);

            tile.set_pixel(x, y, Color4f(c, c, c, 1.0f));

#else

            const size_t src_x = (origin_x + x) * src_tile->get_width() / m_width;
            const size_t src_y = (origin_y + y) * src_tile->get_height() / m_height;

            const AccumulationPixel* pixel =
                reinterpret_cast<const AccumulationPixel*>(
                    src_tile->pixel(src_x, src_y));

            const Color4f color =
                pixel->m_count > 0
                    ? pixel->m_color / static_cast<float>(pixel->m_count)
                    : Color4f(0.0f);

            tile.set_pixel(x, y, color);

#endif
        }
    }
}

}   // namespace renderer
<commit_msg>fixed DEBUG_DISPLAY_SAMPLE_COUNT code path, renamed some variables.<commit_after>
//
// This source file is part of appleseed.
// Visit http://appleseedhq.net/ for additional information and resources.
//
// This software is released under the MIT license.
//
// Copyright (c) 2010-2012 Francois Beaune, Jupiter Jazz
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//

// Interface header.
#include "localaccumulationframebuffer.h"

// appleseed.renderer headers.
#include "renderer/kernel/rendering/sample.h"
#include "renderer/modeling/frame/frame.h"

// appleseed.foundation headers.
#include "foundation/image/canvasproperties.h"
#include "foundation/image/image.h"
#include "foundation/image/pixel.h"
#include "foundation/math/scalar.h"
#include "foundation/platform/compiler.h"
#include "foundation/platform/thread.h"

// Standard headers.
#include <cassert>
#include <algorithm>

using namespace boost;
using namespace foundation;
using namespace std;

namespace renderer
{

// If defined, draw each pixel with a shade of gray proportional to the number of samples it contains.
#undef DEBUG_DISPLAY_SAMPLE_COUNT

LocalAccumulationFramebuffer::LocalAccumulationFramebuffer(
    const size_t    width,
    const size_t    height)
  : AccumulationFramebuffer(width, height)
{
    // todo: change to static_assert<>.
    assert(sizeof(AccumulationPixel) == 5 * sizeof(float));

    const size_t MinSize = 32;

    size_t level_width = width;
    size_t level_height = height;

    do
    {
        m_levels.push_back(
            new Tile(
                max(level_width, MinSize),
                max(level_height, MinSize),
                4 + 1,
                PixelFormatFloat));

        m_set_pixels.push_back(0);

        level_width /= 2;
        level_height /= 2;
    }
    while (level_width > MinSize && level_height > MinSize);

    clear();
}

LocalAccumulationFramebuffer::~LocalAccumulationFramebuffer()
{
    for (size_t i = 0; i < m_levels.size(); ++i)
        delete m_levels[i];
}

void LocalAccumulationFramebuffer::clear()
{
    Spinlock::ScopedLock lock(m_spinlock);

    AccumulationFramebuffer::clear_no_lock();

    for (size_t level_index = 0; level_index < m_levels.size(); ++level_index)
    {
        Tile* level = m_levels[level_index];

        AccumulationPixel* pixel = reinterpret_cast<AccumulationPixel*>(level->pixel(0));
        const size_t pixel_count = level->get_pixel_count();

        for (size_t i = 0; i < pixel_count; ++i)
        {
            pixel[i].m_color.set(0.0f);
            pixel[i].m_count = 0;
        }

        m_set_pixels[level_index] = 0;
    }

    m_current_level = m_levels.size() - 1;
}

void LocalAccumulationFramebuffer::store_samples(
    const size_t    sample_count,
    const Sample    samples[])
{
    Spinlock::ScopedLock lock(m_spinlock);

    const Sample* RESTRICT sample_ptr = samples;
    const Sample* RESTRICT sample_end = samples + sample_count;

    if (m_current_level == 0)
    {
        Tile* level = m_levels[0];

        const double fb_width = static_cast<double>(level->get_width());
        const double fb_height = static_cast<double>(level->get_height());

        while (sample_ptr < sample_end)
        {
            const double fx = sample_ptr->m_position.x * fb_width;
            const double fy = sample_ptr->m_position.y * fb_height;
            const size_t x = truncate<size_t>(fx);
            const size_t y = truncate<size_t>(fy);

            AccumulationPixel* pixel =
                reinterpret_cast<AccumulationPixel*>(level->pixel(x, y));

            pixel->m_color += sample_ptr->m_color;
            pixel->m_count += 1;

            if (pixel->m_count == 1)
                ++m_set_pixels[0];

            ++sample_ptr;
        }
    }
    else
    {
        while (sample_ptr < sample_end)
        {
            for (size_t level_index = 0; level_index <= m_current_level; ++level_index)
            {
                Tile* level = m_levels[level_index];

                const double fx = sample_ptr->m_position.x * level->get_width();
                const double fy = sample_ptr->m_position.y * level->get_height();
                const size_t x = truncate<size_t>(fx);
                const size_t y = truncate<size_t>(fy);

                AccumulationPixel* pixel =
                    reinterpret_cast<AccumulationPixel*>(level->pixel(x, y));

                pixel->m_color += sample_ptr->m_color;
                pixel->m_count += 1;

                if (pixel->m_count == 1)
                {
                    if (++m_set_pixels[level_index] == level->get_pixel_count())
                    {
                        --m_current_level;
                        break;
                    }
                }
            }

            ++sample_ptr;
        }
    }

    m_sample_count += sample_count;
}

void LocalAccumulationFramebuffer::develop_to_frame(Frame& frame) const
{
    Image& image = frame.image();
    const CanvasProperties& frame_props = image.properties();

    assert(frame_props.m_canvas_width == m_width);
    assert(frame_props.m_canvas_height == m_height);
    assert(frame_props.m_channel_count == 4);

    for (size_t ty = 0; ty < frame_props.m_tile_count_y; ++ty)
    {
        for (size_t tx = 0; tx < frame_props.m_tile_count_x; ++tx)
        {
            Tile& tile = image.tile(tx, ty);

            const size_t origin_x = tx * frame_props.m_tile_width;
            const size_t origin_y = ty * frame_props.m_tile_height;

            develop_to_tile(tile, origin_x, origin_y, tx, ty);
        }
    }
}

void LocalAccumulationFramebuffer::develop_to_tile(
    Tile&           tile,
    const size_t    origin_x,
    const size_t    origin_y,
    const size_t    tile_x,
    const size_t    tile_y) const
{
    const size_t tile_width = tile.get_width();
    const size_t tile_height = tile.get_height();

#ifdef DEBUG_DISPLAY_SAMPLE_COUNT

    const Tile* level = m_levels[0];

    for (size_t y = 0; y < tile_height; ++y)
    {
        for (size_t x = 0; x < tile_width; ++x)
        {
            const AccumulationPixel* pixel =
                reinterpret_cast<const AccumulationPixel*>(
                    level->pixel(
                        origin_x + x,
                        origin_y + y));

            const float c = saturate(static_cast<float>(pixel->m_count) / 20.0f);

            tile.set_pixel(x, y, Color4f(c, c, c, 1.0f));
        }
    }

#else

    const size_t display_level_index =
        m_current_level > 0 || m_set_pixels[0] < m_pixel_count
            ? min(m_current_level + 1, m_levels.size() - 1)
            : 0;

    const Tile* level = m_levels[display_level_index];

    for (size_t y = 0; y < tile_height; ++y)
    {
        for (size_t x = 0; x < tile_width; ++x)
        {
            const AccumulationPixel* pixel =
                reinterpret_cast<const AccumulationPixel*>(
                    level->pixel(
                        (origin_x + x) * level->get_width() / m_width,
                        (origin_y + y) * level->get_height() / m_height));

            const Color4f color =
                pixel->m_count > 0
                    ? pixel->m_color / static_cast<float>(pixel->m_count)
                    : Color4f(0.0f);

            tile.set_pixel(x, y, color);
        }
    }

#endif
}

}   // namespace renderer
<|endoftext|>


<commit_before>//----------------------------  work_stream_03.cc  ---------------------------
//    $Id$
//    Version: $Name$
//
//    Copyright (C) 2006, 2013 by the deal.II authors
//
//    This file is subject to QPL and may not be  distributed
//    without copyright and license information. Please refer
//    to the file deal.II/doc/license.html for the  text  and
//    further information on this license.
//
//----------------------------  work_stream_03.cc  ---------------------------

// Moritz originally implemented thread local scratch objects for
// WorkStream in r24748 but it led to failures in the testsuite. what
// exactly went on was a mystery and this test is a first step in
// figuring out what happens by running a simplified version of one of
// the failing tests (deal.II/project_q_01) multiple times and
// verifying that it indeed works

#include "../tests.h"
#include <deal.II/base/logstream.h>
#include <deal.II/base/quadrature_lib.h>
#include <deal.II/base/work_stream.h>
#include <deal.II/lac/vector.h>
#include <deal.II/base/parallel.h>

#include <deal.II/grid/tria.h>
#include <deal.II/dofs/dof_handler.h>
#include <deal.II/grid/grid_generator.h>
#include <deal.II/fe/fe_nothing.h>
#include <deal.II/fe/fe_values.h>

#include <fstream>
#include <vector>

char logname[] = "work_stream_03/output";

template<int dim>
  double
  value(const Point<dim> &p)
  {
    double val = 0;
    for (unsigned int d = 0; d < dim; ++d)
      for (unsigned int i = 0; i <= 1; ++i)
        val += std::pow(p[d], 1. * i);
    return val;
  }

namespace
{
  template<int dim>
    struct Scratch
    {
      Scratch(const FiniteElement<dim> &fe, const Quadrature<dim> &quadrature) :
          fe_collection(fe), quadrature_collection(quadrature), x_fe_values(
              fe_collection, quadrature_collection, update_q_points), rhs_values(
              quadrature_collection.size())
      {
      }

      Scratch(const Scratch &data) :
          fe_collection(data.fe_collection), quadrature_collection(
              data.quadrature_collection), x_fe_values(fe_collection,
              quadrature_collection, update_q_points), rhs_values(
              data.rhs_values)
      {
      }

      const FiniteElement<dim> &fe_collection;
      const Quadrature<dim> &quadrature_collection;

      FEValues<dim> x_fe_values;

      std::vector<double> rhs_values;
    };

  struct CopyData
  {
    std::vector<double> cell_rhs;
  };
}

void
zero_subrange(const unsigned int begin, const unsigned int end,
    std::vector<double> &dst)
{
  for (unsigned int i = begin; i < end; ++i)
    dst[i] = 0;
}

template<int dim>
  void
  mass_assembler(const typename Triangulation<dim>::active_cell_iterator &cell,
      Scratch<dim> &data, CopyData &copy_data)
  {
    data.x_fe_values.reinit(cell);

    const Point<dim> q = data.x_fe_values.quadrature_point(0);

    // this appears to be the key: the following line overwrites some of the memory
    // in which we store the quadrature point location. if the line is moved down,
    // the comparison in the if() always succeeds...
    parallel::apply_to_subranges(0U, copy_data.cell_rhs.size(),
        std_cxx1x::bind(&zero_subrange, std_cxx1x::_1, std_cxx1x::_2,
            std_cxx1x::ref(copy_data.cell_rhs)), 1);

    std::cout << (q != data.x_fe_values.quadrature_point(0) ? '.' : '*')
        << std::flush;

    copy_data.cell_rhs[0] = value(data.x_fe_values.quadrature_point(0));
  }

void
copy_local_to_global(const CopyData &data, double *sum)
{
  *sum += data.cell_rhs[0];
}

void
do_project()
{
  static const int dim = 3;

  Triangulation < dim > triangulation;
  GridGenerator::hyper_cube (triangulation);
  triangulation.refine_global(2);

  FE_Nothing < dim > fe;
  QMidpoint < dim > q;

  for (unsigned int i = 0; i < 12; ++i)
    {
      std::vector<double> tmp;

      double sum = 0;
      Scratch<dim> assembler_data(fe, q);
      CopyData copy_data;
      copy_data.cell_rhs.resize(8);
      WorkStream::run(triangulation.begin_active(), triangulation.end(),
          &mass_assembler<dim>,
          std_cxx1x::bind(&copy_local_to_global, std_cxx1x::_1, &sum),
          assembler_data, copy_data, 8, 1);
      printf("\nCheck: %5.3f\n", sum);
      deallog << sum << std::endl;
    }
}

int main()
{
  std::ofstream logfile(logname);
  deallog << std::setprecision(3);

  deallog.attach(logfile);
  deallog.depth_console(0);
  deallog.threshold_double(1.e-10);

  do_project();
}
<commit_msg>Final version of the test.<commit_after>//----------------------------  work_stream_03.cc  ---------------------------
//    $Id$
//    Version: $Name$
//
//    Copyright (C) 2006, 2013 by the deal.II authors
//
//    This file is subject to QPL and may not be  distributed
//    without copyright and license information. Please refer
//    to the file deal.II/doc/license.html for the  text  and
//    further information on this license.
//
//----------------------------  work_stream_03.cc  ---------------------------

// Moritz originally implemented thread local scratch objects for
// WorkStream in r24748 but it led to failures in the testsuite. what
// exactly went on was a mystery and this test is a first step in
// figuring out what happens by running a simplified version of one of
// the failing tests (deal.II/project_q_01) multiple times and
// verifying that it indeed works

#include "../tests.h"
#include <deal.II/base/logstream.h>
#include <deal.II/base/quadrature_lib.h>
#include <deal.II/base/work_stream.h>
#include <deal.II/lac/vector.h>
#include <deal.II/base/parallel.h>

#include <deal.II/grid/tria.h>
#include <deal.II/dofs/dof_handler.h>
#include <deal.II/grid/grid_generator.h>
#include <deal.II/fe/fe_nothing.h>
#include <deal.II/fe/fe_values.h>

#include <fstream>
#include <vector>

char logname[] = "work_stream_03/output";

template<int dim>
  double
  value(const Point<dim> &p)
  {
    double val = 0;
    for (unsigned int d = 0; d < dim; ++d)
      for (unsigned int i = 0; i <= 1; ++i)
        val += std::pow(p[d], 1. * i);
    return val;
  }

namespace
{
  template<int dim>
    struct Scratch
    {
      Scratch(const FiniteElement<dim> &fe, const Quadrature<dim> &quadrature) :
          fe_collection(fe), quadrature_collection(quadrature), x_fe_values(
              fe_collection, quadrature_collection, update_q_points), rhs_values(
              quadrature_collection.size())
      {
      }

      Scratch(const Scratch &data) :
          fe_collection(data.fe_collection), quadrature_collection(
              data.quadrature_collection), x_fe_values(fe_collection,
              quadrature_collection, update_q_points), rhs_values(
              data.rhs_values)
      {
      }

      const FiniteElement<dim> &fe_collection;
      const Quadrature<dim> &quadrature_collection;

      FEValues<dim> x_fe_values;

      std::vector<double> rhs_values;
    };

  struct CopyData
  {
    std::vector<double> cell_rhs;
  };
}

void
zero_subrange(const unsigned int begin, const unsigned int end,
    std::vector<double> &dst)
{
  for (unsigned int i = begin; i < end; ++i)
    dst[i] = 0;
}

void
zero_element(std::vector<double> &dst,
	     const unsigned int i)
{
  dst[i] = 0;
}

template<int dim>
void
  mass_assembler(const typename Triangulation<dim>::active_cell_iterator &cell,
      Scratch<dim> &data, CopyData &copy_data)
  {
    data.x_fe_values.reinit(cell);

    const Point<dim> q = data.x_fe_values.quadrature_point(0);

    // this appears to be the key: the following two ways both overwrite some
    // of the memory in which we store the quadrature point location.
    parallel::apply_to_subranges(0U, copy_data.cell_rhs.size(),
        std_cxx1x::bind(&zero_subrange, std_cxx1x::_1, std_cxx1x::_2,
            std_cxx1x::ref(copy_data.cell_rhs)), 1);

    Assert(q == data.x_fe_values.quadrature_point(0),
        ExcInternalError());

    copy_data.cell_rhs[0] = value(data.x_fe_values.quadrature_point(0));
  }

void
copy_local_to_global(const CopyData &data, double *sum)
{
  *sum += data.cell_rhs[0];
}

void
do_project()
{
  static const int dim = 3;

  Triangulation < dim > triangulation;
  GridGenerator::hyper_cube (triangulation);
  triangulation.refine_global(2);

  FE_Nothing < dim > fe;
  QMidpoint < dim > q;

  for (unsigned int i = 0; i < 12; ++i)
    {
      std::vector<double> tmp;

      double sum = 0;
      Scratch<dim> assembler_data(fe, q);
      CopyData copy_data;
      copy_data.cell_rhs.resize(8);
      WorkStream::run(triangulation.begin_active(), triangulation.end(),
          &mass_assembler<dim>,
          std_cxx1x::bind(&copy_local_to_global, std_cxx1x::_1, &sum),
          assembler_data, copy_data, 8, 1);

      Assert (std::fabs(sum-288.) < 1e-12, ExcInternalError());
      deallog << sum << std::endl;
    }
}

int main()
{
  std::ofstream logfile(logname);
  deallog << std::setprecision(3);

  deallog.attach(logfile);
  deallog.depth_console(0);
  deallog.threshold_double(1.e-10);

  do_project();
}
<|endoftext|>


<commit_before>/****************************************************************************
 *
 *   Copyright (C) 2019 PX4 Development Team. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name PX4 nor the names of its contributors may be
 *    used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 ****************************************************************************/

#include <px4_arch/io_timer_hw_description.h>

constexpr io_timers_t io_timers[MAX_IO_TIMERS] = {
	initIOTimer(Timer::Timer3, DMA{DMA::Index1, DMA::Stream2, DMA::Channel5}),
	initIOTimer(Timer::Timer1, DMA{DMA::Index2, DMA::Stream5, DMA::Channel6}),
	initIOTimer(Timer::Timer8, DMA{DMA::Index2, DMA::Stream1, DMA::Channel7}),
	initIOTimer(Timer::Timer5, DMA{DMA::Index1, DMA::Stream6, DMA::Channel6}),
};

constexpr timer_io_channels_t timer_io_channels[MAX_TIMER_IO_CHANNELS] = {
	initIOTimerChannel(io_timers, {Timer::Timer5, Timer::Channel4}, {GPIO::PortA, GPIO::Pin3}),
	initIOTimerChannel(io_timers, {Timer::Timer8, Timer::Channel4}, {GPIO::PortC, GPIO::Pin9}),
	initIOTimerChannel(io_timers, {Timer::Timer1, Timer::Channel2}, {GPIO::PortE, GPIO::Pin11}),
	initIOTimerChannel(io_timers, {Timer::Timer1, Timer::Channel1}, {GPIO::PortE, GPIO::Pin9}),
	initIOTimerChannel(io_timers, {Timer::Timer3, Timer::Channel4}, {GPIO::PortB, GPIO::Pin1}),
	initIOTimerChannel(io_timers, {Timer::Timer3, Timer::Channel3}, {GPIO::PortB, GPIO::Pin0}),
};

constexpr io_timers_channel_mapping_t io_timers_channel_mapping =
	initIOTimerChannelMapping(io_timers, timer_io_channels);
<commit_msg>kakutef7: fix output ordering<commit_after>/****************************************************************************
 *
 *   Copyright (C) 2019 PX4 Development Team. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name PX4 nor the names of its contributors may be
 *    used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 ****************************************************************************/

#include <px4_arch/io_timer_hw_description.h>

constexpr io_timers_t io_timers[MAX_IO_TIMERS] = {
	initIOTimer(Timer::Timer3, DMA{DMA::Index1, DMA::Stream2, DMA::Channel5}),
	initIOTimer(Timer::Timer1, DMA{DMA::Index2, DMA::Stream5, DMA::Channel6}),
	initIOTimer(Timer::Timer8, DMA{DMA::Index2, DMA::Stream1, DMA::Channel7}),
	initIOTimer(Timer::Timer5, DMA{DMA::Index1, DMA::Stream6, DMA::Channel6}),
};

constexpr timer_io_channels_t timer_io_channels[MAX_TIMER_IO_CHANNELS] = {
	initIOTimerChannel(io_timers, {Timer::Timer3, Timer::Channel3}, {GPIO::PortB, GPIO::Pin0}),
	initIOTimerChannel(io_timers, {Timer::Timer3, Timer::Channel4}, {GPIO::PortB, GPIO::Pin1}),
	initIOTimerChannel(io_timers, {Timer::Timer1, Timer::Channel1}, {GPIO::PortE, GPIO::Pin9}),
	initIOTimerChannel(io_timers, {Timer::Timer1, Timer::Channel2}, {GPIO::PortE, GPIO::Pin11}),
	initIOTimerChannel(io_timers, {Timer::Timer8, Timer::Channel4}, {GPIO::PortC, GPIO::Pin9}),
	initIOTimerChannel(io_timers, {Timer::Timer5, Timer::Channel4}, {GPIO::PortA, GPIO::Pin3}),
};

constexpr io_timers_channel_mapping_t io_timers_channel_mapping =
	initIOTimerChannelMapping(io_timers, timer_io_channels);
<|endoftext|>


<commit_before>#include "hand.h"
#include "card.h"
#include "gameExceptions.h"

#include <vector>

Hand::Hand()
{
    _cards = std::vector<Card>();
}

Hand::Hand( std::vector<Card> cards )
{
    _cards = std::vector<Card>();
    _cards.reserve(cards.size());
    _cards = cards;
}

unsigned int Hand::GetScore()
{
    unsigned int score = 0;

    for (std::vector<Card>::const_iterator card = _cards.begin(); card != _cards.end(); ++card)
    {
        score += card->GetScore();
    }

    return score;
}

void Hand::AddCard(Card card)
{
    if (card.IsValid())
    {
        _cards.push_back(card);
    }
    else
    {
        throw InvalidCardException("Invalid card at Hand::AddCard(Card card)!");
    }
}

<commit_msg>Added verification for the Aces cards in the GetScore() method of the hand class.<commit_after>#include "hand.h"
#include "card.h"
#include "gameExceptions.h"

#include <vector>

Hand::Hand()
{
    _cards = std::vector<Card>();
}

Hand::Hand( std::vector<Card> cards )
{
    _cards = std::vector<Card>();
    _cards.reserve(cards.size());
    _cards = cards;
}

unsigned int Hand::GetScore()
{
    unsigned int score = 0;
    bool hasAce = false;
    for (std::vector<Card>::const_iterator card = _cards.begin(); card != _cards.end(); ++card)
    {
        if (card->GetName() !=  Ace)
            score += card->GetScore();
        else
            hasAce = true;
    }

    if (hasAce)
        for (std::vector<Card>::iterator card = _cards.begin(); card != _cards.end(); ++card)
        {
            if (card->GetName() ==  Ace)
                if (score + 11 > 21)
                {
                    card->SetScore(11);
                    score += 11;
                }
                else
                {
                    card->SetScore(1);
                    score += 1;
                }
        }

    return score;
}

void Hand::AddCard(Card card)
{
    if (card.IsValid())
    {
        _cards.push_back(card);
    }
    else
    {
        throw InvalidCardException("Invalid card at Hand::AddCard(Card card)!");
    }
}

<|endoftext|>