<commit_before>#include "xchainer/routines/sorting.h"

#include <cstdint>
#include <string>

#include <gtest/gtest.h>
#include <nonstd/optional.hpp>

#include "xchainer/array.h"
#include "xchainer/device.h"
#include "xchainer/device_id.h"
#include "xchainer/error.h"
#include "xchainer/testing/array.h"
#include "xchainer/testing/array_check.h"
#include "xchainer/testing/device_session.h"
#include "xchainer/testing/routines.h"

namespace xchainer {
namespace {

class SortingTest : public ::testing::TestWithParam<std::string> {
protected:
    void SetUp() override {
        const std::string& backend_name = GetParam();
        device_session_.emplace(DeviceId{backend_name, 0});
    }

    void TearDown() override { device_session_.reset(); }

private:
    nonstd::optional<testing::DeviceSession> device_session_;
};

TEST_P(SortingTest, ArgMax) {
    Array a = testing::BuildArray({2, 3}).WithData<float>({1, 4, 3, 0, 1, 4}).WithPadding(1);
    Array e = testing::BuildArray({3}).WithData<int64_t>({0, 0, 1});

    testing::CheckForward(
            [](const std::vector<Array>& xs) { return std::vector<Array>{ArgMax(xs[0], 0)}; },
            {a},
            {e},
            // TODO(sonots): Run concurrency test in CUDA
            GetParam() == "cuda" ? 0 : 1);
}

TEST_P(SortingTest, ArgMaxNegativeAxis) {
    Array a = testing::BuildArray({2, 3}).WithData<float>({1, 4, 3, 0, 1, 4});
    Array e = testing::BuildArray({2}).WithData<int64_t>({1, 2});

    testing::CheckForward(
            [](const std::vector<Array>& xs) { return std::vector<Array>{ArgMax(xs[0], -1)}; },
            {a},
            {e},
            // TODO(sonots): Run concurrency test in CUDA
            GetParam() == "cuda" ? 0 : 1);
}

TEST_P(SortingTest, ArgMaxAllAxes) {
    Array a = testing::BuildArray({2, 3}).WithData<float>({1, 4, 3, 0, 1, 4});
    Array e = testing::BuildArray({}).WithData<int64_t>({1});

    testing::CheckForward(
            [](const std::vector<Array>& xs) { return std::vector<Array>{ArgMax(xs[0])}; },
            {a},
            {e},
            // TODO(sonots): Run concurrency test in CUDA
            GetParam() == "cuda" ? 0 : 1);
}

TEST_P(SortingTest, ArgMaxInvalidAxis) {
    Array a = testing::BuildArray({2, 3}).WithData<float>({1, 4, 3, 0, 1, 4});
    EXPECT_THROW(ArgMax(a, 3), DimensionError);
}

TEST_P(SortingTest, ArgMaxEmpty) {
    Array a = Zeros({0}, Dtype::kFloat32);
    EXPECT_THROW(ArgMax(a), DimensionError);
}

INSTANTIATE_TEST_CASE_P(
        ForEachBackend,
        SortingTest,
        ::testing::Values(
#ifdef XCHAINER_ENABLE_CUDA
                std::string{"cuda"},
#endif  // XCHAINER_ENABLE_CUDA
                std::string{"native"}));

}  // namespace
}  // namespace xchainer
<commit_msg>cpplint<commit_after>#include "xchainer/routines/sorting.h"

#include <cstdint>
#include <string>
#include <vector>

#include <gtest/gtest.h>
#include <nonstd/optional.hpp>

#include "xchainer/array.h"
#include "xchainer/device.h"
#include "xchainer/device_id.h"
#include "xchainer/error.h"
#include "xchainer/testing/array.h"
#include "xchainer/testing/array_check.h"
#include "xchainer/testing/device_session.h"
#include "xchainer/testing/routines.h"

namespace xchainer {
namespace {

class SortingTest : public ::testing::TestWithParam<std::string> {
protected:
    void SetUp() override {
        const std::string& backend_name = GetParam();
        device_session_.emplace(DeviceId{backend_name, 0});
    }

    void TearDown() override { device_session_.reset(); }

private:
    nonstd::optional<testing::DeviceSession> device_session_;
};

TEST_P(SortingTest, ArgMax) {
    Array a = testing::BuildArray({2, 3}).WithData<float>({1, 4, 3, 0, 1, 4}).WithPadding(1);
    Array e = testing::BuildArray({3}).WithData<int64_t>({0, 0, 1});

    testing::CheckForward(
            [](const std::vector<Array>& xs) { return std::vector<Array>{ArgMax(xs[0], 0)}; },
            {a},
            {e},
            // TODO(sonots): Run concurrency test in CUDA
            GetParam() == "cuda" ? 0 : 1);
}

TEST_P(SortingTest, ArgMaxNegativeAxis) {
    Array a = testing::BuildArray({2, 3}).WithData<float>({1, 4, 3, 0, 1, 4});
    Array e = testing::BuildArray({2}).WithData<int64_t>({1, 2});

    testing::CheckForward(
            [](const std::vector<Array>& xs) { return std::vector<Array>{ArgMax(xs[0], -1)}; },
            {a},
            {e},
            // TODO(sonots): Run concurrency test in CUDA
            GetParam() == "cuda" ? 0 : 1);
}

TEST_P(SortingTest, ArgMaxAllAxes) {
    Array a = testing::BuildArray({2, 3}).WithData<float>({1, 4, 3, 0, 1, 4});
    Array e = testing::BuildArray({}).WithData<int64_t>({1});

    testing::CheckForward(
            [](const std::vector<Array>& xs) { return std::vector<Array>{ArgMax(xs[0])}; },
            {a},
            {e},
            // TODO(sonots): Run concurrency test in CUDA
            GetParam() == "cuda" ? 0 : 1);
}

TEST_P(SortingTest, ArgMaxInvalidAxis) {
    Array a = testing::BuildArray({2, 3}).WithData<float>({1, 4, 3, 0, 1, 4});
    EXPECT_THROW(ArgMax(a, 3), DimensionError);
}

TEST_P(SortingTest, ArgMaxEmpty) {
    Array a = Zeros({0}, Dtype::kFloat32);
    EXPECT_THROW(ArgMax(a), DimensionError);
}

INSTANTIATE_TEST_CASE_P(
        ForEachBackend,
        SortingTest,
        ::testing::Values(
#ifdef XCHAINER_ENABLE_CUDA
                std::string{"cuda"},
#endif  // XCHAINER_ENABLE_CUDA
                std::string{"native"}));

}  // namespace
}  // namespace xchainer
<|endoftext|>


<commit_before>// Copyright 2022 The XLS Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "absl/status/status.h"
#include "clang/include/clang/AST/Expr.h"
#include "clang/include/clang/AST/ExprCXX.h"
#include "xls/common/status/status_macros.h"
#include "xls/contrib/xlscc/translator.h"

namespace xlscc {

absl::StatusOr<IOOp*> Translator::AddOpToChannel(IOOp& op, IOChannel* channel,
                                                 const xls::SourceInfo& loc,
                                                 bool mask) {
  context().any_side_effects_requested = true;

  if (context().mask_side_effects || mask) {
    IOOpReturn ret;
    ret.generate_expr = false;
    XLS_ASSIGN_OR_RETURN(xls::BValue default_bval,
                         CreateDefaultValue(channel->item_type, loc));
    op.input_value = CValue(default_bval, channel->item_type);
    return &op;
  }
  XLS_CHECK_NE(channel, nullptr);
  XLS_CHECK_EQ(op.channel, nullptr);
  op.channel_op_index = channel->total_ops++;
  op.channel = channel;
  op.op_location = loc;

  // Operation type is added late, as it's only known from the read()/write()
  // call(s)
  if (channel->channel_op_type == OpType::kNull) {
    channel->channel_op_type = op.op;
  } else {
    if (channel->channel_op_type != op.op) {
      return absl::UnimplementedError(
          ErrorMessage(loc, "Channels should be either input or output"));
    }
  }

  // Channel must be inserted first by AddOpToChannel
  if (op.op == OpType::kRecv) {
    xls::Type* xls_item_type;
    std::shared_ptr<CType> channel_item_type;
    if (op.is_blocking) {
      XLS_ASSIGN_OR_RETURN(xls_item_type,
                           TranslateTypeToXLS(channel->item_type, loc));
      channel_item_type = channel->item_type;
    } else {
      XLS_ASSIGN_OR_RETURN(xls::Type * primary_item_type,
                           TranslateTypeToXLS(channel->item_type, loc));
      xls_item_type =
          package_->GetTupleType({primary_item_type, package_->GetBitsType(1)});

      channel_item_type =
          std::make_shared<CInternalTuple>(std::vector<std::shared_ptr<CType>>(
              {channel->item_type, std::make_shared<CBoolType>()}));
    }
    const int64_t channel_op_index = op.channel_op_index;

    std::string safe_param_name =
        absl::StrFormat("%s_op%i", op.channel->unique_name, channel_op_index);

    xls::BValue pbval =
        context().fb->Param(safe_param_name, xls_item_type, loc);

    // Check for duplicate params
    if (!pbval.valid()) {
      return absl::InternalError(ErrorMessage(
          loc,
          "Failed to create implicit parameter %s, duplicate? See b/239861050",
          safe_param_name.c_str()));
    }

    op.input_value = CValue(pbval, channel_item_type);
  }

  context().sf->io_ops.push_back(op);

  if (op.op == OpType::kRecv) {
    SideEffectingParameter side_effecting_param;
    side_effecting_param.type = SideEffectingParameterType::kIOOp;
    side_effecting_param.param_name =
        op.input_value.rvalue().node()->As<xls::Param>()->GetName();
    side_effecting_param.io_op = &context().sf->io_ops.back();
    context().sf->side_effecting_parameters.push_back(side_effecting_param);
  }

  return &context().sf->io_ops.back();
}

absl::StatusOr<std::shared_ptr<CType>> Translator::GetChannelType(
    const clang::ParmVarDecl* channel_param, const xls::SourceInfo& loc) {
  XLS_ASSIGN_OR_RETURN(StrippedType stripped,
                       StripTypeQualifiers(channel_param->getType()));
  auto template_spec = clang::dyn_cast<const clang::TemplateSpecializationType>(
      stripped.base.getTypePtr());
  if (template_spec == nullptr) {
    return absl::UnimplementedError(
        ErrorMessage(loc, "Channel type should be a template specialization"));
  }
  if (template_spec->getNumArgs() != 1) {
    return absl::UnimplementedError(
        ErrorMessage(loc, "Channel should have 1 template args"));
  }
  const clang::TemplateArgument& arg = template_spec->getArg(0);
  return TranslateTypeFromClang(arg.getAsType(), loc);
}

absl::Status Translator::CreateChannelParam(
    const clang::ParmVarDecl* channel_param, const xls::SourceInfo& loc) {
  XLS_CHECK(!context().variables.contains(channel_param));

  XLS_ASSIGN_OR_RETURN(std::shared_ptr<CType> ctype,
                       GetChannelType(channel_param, loc));

  IOChannel new_channel;

  new_channel.item_type = ctype;
  new_channel.unique_name = channel_param->getNameAsString();

  context().sf->io_channels.push_back(new_channel);

  auto channel_type = std::make_shared<CChannelType>(ctype);
  auto lvalue = std::make_shared<LValue>(&context().sf->io_channels.back());
  XLS_RETURN_IF_ERROR(
      DeclareVariable(channel_param,
                      CValue(/*rvalue=*/xls::BValue(), channel_type,
                             /*disable_type_check=*/true, lvalue),
                      loc));

  return absl::OkStatus();
}

absl::StatusOr<Translator::IOOpReturn> Translator::InterceptIOOp(
    const clang::Expr* expr, const xls::SourceInfo& loc) {
  if (auto member_call =
          clang::dyn_cast<const clang::CXXMemberCallExpr>(expr)) {
    const clang::Expr* object = member_call->getImplicitObjectArgument();

    XLS_ASSIGN_OR_RETURN(IOChannel * channel, GetChannelForExprOrNull(object));

    if (channel != nullptr) {
      const clang::FunctionDecl* funcdecl = member_call->getDirectCallee();
      const std::string op_name = funcdecl->getNameAsString();

      xls::BValue op_condition = context().full_condition_bval(loc);
      XLS_CHECK(op_condition.valid());

      // Short circuit the op condition if possible
      XLS_RETURN_IF_ERROR(ShortCircuitBVal(op_condition, loc));

      // Ignore IO ops that are definitely condition = 0
      // XLS opt also does this down-stream, but we try to do it here
      // for cases like "if(constexpr) {ch.read();} else {ch.write();}
      // which otherwise confuse XLS[cc] itself.
      bool do_default = false;

      absl::StatusOr<xls::Value> eval_result =
          EvaluateBVal(op_condition, loc, /*do_check=*/false);
      if (eval_result.ok()) {
        if (eval_result.value().IsAllZeros()) {
          do_default = true;
        }
      }

      auto call = clang::dyn_cast<const clang::CallExpr>(expr);

      IOOpReturn ret;
      ret.generate_expr = false;

      IOOp op;
      const clang::Expr* assign_ret_value_to = nullptr;

      if (op_name == "read") {
        if (call->getNumArgs() == 1) {
          assign_ret_value_to = call->getArg(0);
        } else if (call->getNumArgs() != 0) {
          return absl::UnimplementedError(ErrorMessage(
              loc, "IO read() should have one or zero argument(s)"));
        }
        op.op = OpType::kRecv;
        op.ret_value = op_condition;
        op.is_blocking = true;
      } else if (op_name == "nb_read") {
        if (call->getNumArgs() != 1) {
          return absl::UnimplementedError(
              ErrorMessage(loc, "IO nb_read() should have one argument"));
        }
        assign_ret_value_to = call->getArg(0);
        op.op = OpType::kRecv;
        op.ret_value = op_condition;
        op.is_blocking = false;
      } else if (op_name == "write") {
        if (call->getNumArgs() != 1) {
          return absl::UnimplementedError(
              ErrorMessage(loc, "IO write() should have one argument"));
        }

        XLS_ASSIGN_OR_RETURN(CValue out_val,
                             GenerateIR_Expr(call->getArg(0), loc));

        std::vector<xls::BValue> sp = {out_val.rvalue(), op_condition};
        op.ret_value = context().fb->Tuple(sp, loc);
        op.op = OpType::kSend;
        op.is_blocking = true;
      } else {
        return absl::UnimplementedError(
            ErrorMessage(loc, "Unsupported IO op: %s", op_name));
      }

      XLS_ASSIGN_OR_RETURN(
          IOOp * op_ptr, AddOpToChannel(op, channel, loc, /*mask=*/do_default));
      (void)op_ptr;

      ret.value = op.input_value;
      if (assign_ret_value_to != nullptr) {
        if (op.is_blocking) {
          XLS_RETURN_IF_ERROR(Assign(assign_ret_value_to, ret.value, loc));
        } else {
          xls::BValue read_ready =
              context().fb->TupleIndex(ret.value.rvalue(), 1);
          xls::BValue read_value =
              context().fb->TupleIndex(ret.value.rvalue(), 0);

          if (!ret.value.type()->Is<CInternalTuple>()) {
            return absl::UnimplementedError(
                ErrorMessage(loc, "Unsupported IO op: %s", op_name));
          }

          CValue ret_object_value;
          ret_object_value = CValue(
              read_value, ret.value.type()->As<CInternalTuple>()->fields()[0]);

          {
            PushContextGuard condition_guard(*this, loc);
            XLS_RETURN_IF_ERROR(and_condition(read_ready, loc));
            XLS_RETURN_IF_ERROR(Assign(
              assign_ret_value_to, ret_object_value, loc));
          }
          CValue ret_struct = CValue(read_ready, std::make_shared<CBoolType>());
          ret.value = ret_struct;
        }
      }
      return ret;
    }
  }

  IOOpReturn ret;
  ret.generate_expr = true;
  return ret;
}

absl::Status Translator::ExtractExternalIOChannelDecls() {
  for (const auto& [decl, cval] : context().variables) {
    auto channel_type = dynamic_cast<const CChannelType*>(cval.type().get());
    if (channel_type == nullptr) {
      continue;
    }
    XLS_CHECK_NE(cval.lvalue(), nullptr);
    XLS_CHECK_NE(cval.lvalue()->channel_leaf(), nullptr);
    context().sf->io_channels_by_decl[decl] = cval.lvalue()->channel_leaf();
    context().sf->decls_by_io_channel[cval.lvalue()->channel_leaf()] = decl;
  }
  return absl::OkStatus();
}

}  // namespace xlscc
<commit_msg>Integrate LLVM at llvm/llvm-project@fdac4c4e92e5<commit_after>// Copyright 2022 The XLS Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "absl/status/status.h"
#include "clang/include/clang/AST/Expr.h"
#include "clang/include/clang/AST/ExprCXX.h"
#include "xls/common/status/status_macros.h"
#include "xls/contrib/xlscc/translator.h"

namespace xlscc {

absl::StatusOr<IOOp*> Translator::AddOpToChannel(IOOp& op, IOChannel* channel,
                                                 const xls::SourceInfo& loc,
                                                 bool mask) {
  context().any_side_effects_requested = true;

  if (context().mask_side_effects || mask) {
    IOOpReturn ret;
    ret.generate_expr = false;
    XLS_ASSIGN_OR_RETURN(xls::BValue default_bval,
                         CreateDefaultValue(channel->item_type, loc));
    op.input_value = CValue(default_bval, channel->item_type);
    return &op;
  }
  XLS_CHECK_NE(channel, nullptr);
  XLS_CHECK_EQ(op.channel, nullptr);
  op.channel_op_index = channel->total_ops++;
  op.channel = channel;
  op.op_location = loc;

  // Operation type is added late, as it's only known from the read()/write()
  // call(s)
  if (channel->channel_op_type == OpType::kNull) {
    channel->channel_op_type = op.op;
  } else {
    if (channel->channel_op_type != op.op) {
      return absl::UnimplementedError(
          ErrorMessage(loc, "Channels should be either input or output"));
    }
  }

  // Channel must be inserted first by AddOpToChannel
  if (op.op == OpType::kRecv) {
    xls::Type* xls_item_type;
    std::shared_ptr<CType> channel_item_type;
    if (op.is_blocking) {
      XLS_ASSIGN_OR_RETURN(xls_item_type,
                           TranslateTypeToXLS(channel->item_type, loc));
      channel_item_type = channel->item_type;
    } else {
      XLS_ASSIGN_OR_RETURN(xls::Type * primary_item_type,
                           TranslateTypeToXLS(channel->item_type, loc));
      xls_item_type =
          package_->GetTupleType({primary_item_type, package_->GetBitsType(1)});

      channel_item_type =
          std::make_shared<CInternalTuple>(std::vector<std::shared_ptr<CType>>(
              {channel->item_type, std::make_shared<CBoolType>()}));
    }
    const int64_t channel_op_index = op.channel_op_index;

    std::string safe_param_name =
        absl::StrFormat("%s_op%i", op.channel->unique_name, channel_op_index);

    xls::BValue pbval =
        context().fb->Param(safe_param_name, xls_item_type, loc);

    // Check for duplicate params
    if (!pbval.valid()) {
      return absl::InternalError(ErrorMessage(
          loc,
          "Failed to create implicit parameter %s, duplicate? See b/239861050",
          safe_param_name.c_str()));
    }

    op.input_value = CValue(pbval, channel_item_type);
  }

  context().sf->io_ops.push_back(op);

  if (op.op == OpType::kRecv) {
    SideEffectingParameter side_effecting_param;
    side_effecting_param.type = SideEffectingParameterType::kIOOp;
    side_effecting_param.param_name =
        op.input_value.rvalue().node()->As<xls::Param>()->GetName();
    side_effecting_param.io_op = &context().sf->io_ops.back();
    context().sf->side_effecting_parameters.push_back(side_effecting_param);
  }

  return &context().sf->io_ops.back();
}

absl::StatusOr<std::shared_ptr<CType>> Translator::GetChannelType(
    const clang::ParmVarDecl* channel_param, const xls::SourceInfo& loc) {
  XLS_ASSIGN_OR_RETURN(StrippedType stripped,
                       StripTypeQualifiers(channel_param->getType()));
  auto template_spec = clang::dyn_cast<const clang::TemplateSpecializationType>(
      stripped.base.getTypePtr());
  if (template_spec == nullptr) {
    return absl::UnimplementedError(
        ErrorMessage(loc, "Channel type should be a template specialization"));
  }
  if (template_spec->template_arguments().size() != 1) {
    return absl::UnimplementedError(
        ErrorMessage(loc, "Channel should have 1 template args"));
  }
  const clang::TemplateArgument& arg = template_spec->template_arguments()[0];
  return TranslateTypeFromClang(arg.getAsType(), loc);
}

absl::Status Translator::CreateChannelParam(
    const clang::ParmVarDecl* channel_param, const xls::SourceInfo& loc) {
  XLS_CHECK(!context().variables.contains(channel_param));

  XLS_ASSIGN_OR_RETURN(std::shared_ptr<CType> ctype,
                       GetChannelType(channel_param, loc));

  IOChannel new_channel;

  new_channel.item_type = ctype;
  new_channel.unique_name = channel_param->getNameAsString();

  context().sf->io_channels.push_back(new_channel);

  auto channel_type = std::make_shared<CChannelType>(ctype);
  auto lvalue = std::make_shared<LValue>(&context().sf->io_channels.back());
  XLS_RETURN_IF_ERROR(
      DeclareVariable(channel_param,
                      CValue(/*rvalue=*/xls::BValue(), channel_type,
                             /*disable_type_check=*/true, lvalue),
                      loc));

  return absl::OkStatus();
}

absl::StatusOr<Translator::IOOpReturn> Translator::InterceptIOOp(
    const clang::Expr* expr, const xls::SourceInfo& loc) {
  if (auto member_call =
          clang::dyn_cast<const clang::CXXMemberCallExpr>(expr)) {
    const clang::Expr* object = member_call->getImplicitObjectArgument();

    XLS_ASSIGN_OR_RETURN(IOChannel * channel, GetChannelForExprOrNull(object));

    if (channel != nullptr) {
      const clang::FunctionDecl* funcdecl = member_call->getDirectCallee();
      const std::string op_name = funcdecl->getNameAsString();

      xls::BValue op_condition = context().full_condition_bval(loc);
      XLS_CHECK(op_condition.valid());

      // Short circuit the op condition if possible
      XLS_RETURN_IF_ERROR(ShortCircuitBVal(op_condition, loc));

      // Ignore IO ops that are definitely condition = 0
      // XLS opt also does this down-stream, but we try to do it here
      // for cases like "if(constexpr) {ch.read();} else {ch.write();}
      // which otherwise confuse XLS[cc] itself.
      bool do_default = false;

      absl::StatusOr<xls::Value> eval_result =
          EvaluateBVal(op_condition, loc, /*do_check=*/false);
      if (eval_result.ok()) {
        if (eval_result.value().IsAllZeros()) {
          do_default = true;
        }
      }

      auto call = clang::dyn_cast<const clang::CallExpr>(expr);

      IOOpReturn ret;
      ret.generate_expr = false;

      IOOp op;
      const clang::Expr* assign_ret_value_to = nullptr;

      if (op_name == "read") {
        if (call->getNumArgs() == 1) {
          assign_ret_value_to = call->getArg(0);
        } else if (call->getNumArgs() != 0) {
          return absl::UnimplementedError(ErrorMessage(
              loc, "IO read() should have one or zero argument(s)"));
        }
        op.op = OpType::kRecv;
        op.ret_value = op_condition;
        op.is_blocking = true;
      } else if (op_name == "nb_read") {
        if (call->getNumArgs() != 1) {
          return absl::UnimplementedError(
              ErrorMessage(loc, "IO nb_read() should have one argument"));
        }
        assign_ret_value_to = call->getArg(0);
        op.op = OpType::kRecv;
        op.ret_value = op_condition;
        op.is_blocking = false;
      } else if (op_name == "write") {
        if (call->getNumArgs() != 1) {
          return absl::UnimplementedError(
              ErrorMessage(loc, "IO write() should have one argument"));
        }

        XLS_ASSIGN_OR_RETURN(CValue out_val,
                             GenerateIR_Expr(call->getArg(0), loc));

        std::vector<xls::BValue> sp = {out_val.rvalue(), op_condition};
        op.ret_value = context().fb->Tuple(sp, loc);
        op.op = OpType::kSend;
        op.is_blocking = true;
      } else {
        return absl::UnimplementedError(
            ErrorMessage(loc, "Unsupported IO op: %s", op_name));
      }

      XLS_ASSIGN_OR_RETURN(
          IOOp * op_ptr, AddOpToChannel(op, channel, loc, /*mask=*/do_default));
      (void)op_ptr;

      ret.value = op.input_value;
      if (assign_ret_value_to != nullptr) {
        if (op.is_blocking) {
          XLS_RETURN_IF_ERROR(Assign(assign_ret_value_to, ret.value, loc));
        } else {
          xls::BValue read_ready =
              context().fb->TupleIndex(ret.value.rvalue(), 1);
          xls::BValue read_value =
              context().fb->TupleIndex(ret.value.rvalue(), 0);

          if (!ret.value.type()->Is<CInternalTuple>()) {
            return absl::UnimplementedError(
                ErrorMessage(loc, "Unsupported IO op: %s", op_name));
          }

          CValue ret_object_value;
          ret_object_value = CValue(
              read_value, ret.value.type()->As<CInternalTuple>()->fields()[0]);

          {
            PushContextGuard condition_guard(*this, loc);
            XLS_RETURN_IF_ERROR(and_condition(read_ready, loc));
            XLS_RETURN_IF_ERROR(Assign(
              assign_ret_value_to, ret_object_value, loc));
          }
          CValue ret_struct = CValue(read_ready, std::make_shared<CBoolType>());
          ret.value = ret_struct;
        }
      }
      return ret;
    }
  }

  IOOpReturn ret;
  ret.generate_expr = true;
  return ret;
}

absl::Status Translator::ExtractExternalIOChannelDecls() {
  for (const auto& [decl, cval] : context().variables) {
    auto channel_type = dynamic_cast<const CChannelType*>(cval.type().get());
    if (channel_type == nullptr) {
      continue;
    }
    XLS_CHECK_NE(cval.lvalue(), nullptr);
    XLS_CHECK_NE(cval.lvalue()->channel_leaf(), nullptr);
    context().sf->io_channels_by_decl[decl] = cval.lvalue()->channel_leaf();
    context().sf->decls_by_io_channel[cval.lvalue()->channel_leaf()] = decl;
  }
  return absl::OkStatus();
}

}  // namespace xlscc
<|endoftext|>


<commit_before>/*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
 *
 *  Gearmand client and server library.
 *
 *  Copyright (C) 2012 Data Differential, http://datadifferential.com/
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are
 *  met:
 *
 *      * Redistributions of source code must retain the above copyright
 *  notice, this list of conditions and the following disclaimer.
 *
 *      * Redistributions in binary form must reproduce the above
 *  copyright notice, this list of conditions and the following disclaimer
 *  in the documentation and/or other materials provided with the
 *  distribution.
 *
 *      * The names of its contributors may not be used to endorse or
 *  promote products derived from this software without specific prior
 *  written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include "gear_config.h"
#include <libgearman/common.h>

#include <cstring>

namespace libgearman {
namespace protocol {

static inline gearman_return_t __submit(gearman_task_st& task,
                                        const gearman_command_t command,
                                        const gearman_string_t &function,
                                        const gearman_string_t &workload)
{
  const void *args[3];
  size_t args_size[3];

  /**
    @todo fix it so that NULL is done by default by the API not by happenstance.
  */
  char function_buffer[1024];
  if (task.client->universal._namespace)
  {
    char *ptr= function_buffer;
    memcpy(ptr, gearman_string_value(task.client->universal._namespace), gearman_string_length(task.client->universal._namespace));
    ptr+= gearman_string_length(task.client->universal._namespace);

    memcpy(ptr, gearman_c_str(function), gearman_size(function) +1);
    ptr+= gearman_size(function);

    args[0]= function_buffer;
    args_size[0]= ptr -function_buffer +1;
  }
  else
  {
    args[0]= gearman_c_str(function);
    args_size[0]= gearman_size(function) +1;
  }

  args[1]= task.unique;
  args_size[1]= task.unique_length +1;

  args[2]= gearman_c_str(workload);
  args_size[2]= gearman_size(workload);

  return gearman_packet_create_args(task.client->universal, task.send,
                                    GEARMAN_MAGIC_REQUEST,
                                    command,
                                    args, args_size,
                                    3);
}

gearman_return_t submit(gearman_task_st& task,
                        const gearman_command_t command,
                        const gearman_string_t &function,
                        const gearman_string_t &workload)
{
  return __submit(task, command, function, workload);
}

gearman_return_t submit_background(gearman_task_st& task,
                                   const gearman_command_t command,
                                   const gearman_string_t &function,
                                   const gearman_string_t &workload)
{
  return __submit(task, command, function, workload);
}

gearman_return_t submit_epoch(gearman_task_st& task,
                              const gearman_string_t &function,
                              const gearman_string_t &workload,
                              time_t when)
{
  const void *args[4];
  size_t args_size[4];

  /**
    @todo fix it so that NULL is done by default by the API not by happenstance.
  */
  char function_buffer[1024];
  if (task.client->universal._namespace)
  {
    char *ptr= function_buffer;
    memcpy(ptr, gearman_string_value(task.client->universal._namespace), gearman_string_length(task.client->universal._namespace));
    ptr+= gearman_string_length(task.client->universal._namespace);

    memcpy(ptr, gearman_c_str(function), gearman_size(function) +1);
    ptr+= gearman_size(function);

    args[0]= function_buffer;
    args_size[0]= ptr -function_buffer +1;
  }
  else
  {
    args[0]= gearman_c_str(function);
    args_size[0]= gearman_size(function) +1;
  }

  args[1]= task.unique;
  args_size[1]= task.unique_length +1;

  char time_string[30];
  int length= snprintf(time_string, sizeof(time_string), "%" PRIu64, static_cast<int64_t>(when));
  args[2]= time_string;
  args_size[2]= length +1;

  args[3]= gearman_c_str(workload);
  args_size[3]= gearman_size(workload);

  return gearman_packet_create_args(task.client->universal, task.send,
                                    GEARMAN_MAGIC_REQUEST,
                                    GEARMAN_COMMAND_SUBMIT_JOB_EPOCH,
                                    args, args_size,
                                    4);
}

} // namespace protocol
} // namespace libgearman

<commit_msg>Missing stdio which is required for snprintf.<commit_after>/*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
 *
 *  Gearmand client and server library.
 *
 *  Copyright (C) 2012 Data Differential, http://datadifferential.com/
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are
 *  met:
 *
 *      * Redistributions of source code must retain the above copyright
 *  notice, this list of conditions and the following disclaimer.
 *
 *      * Redistributions in binary form must reproduce the above
 *  copyright notice, this list of conditions and the following disclaimer
 *  in the documentation and/or other materials provided with the
 *  distribution.
 *
 *      * The names of its contributors may not be used to endorse or
 *  promote products derived from this software without specific prior
 *  written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include "gear_config.h"
#include <libgearman/common.h>

#include <cstdio>
#include <cstring>

namespace libgearman {
namespace protocol {

static inline gearman_return_t __submit(gearman_task_st& task,
                                        const gearman_command_t command,
                                        const gearman_string_t &function,
                                        const gearman_string_t &workload)
{
  const void *args[3];
  size_t args_size[3];

  /**
    @todo fix it so that NULL is done by default by the API not by happenstance.
  */
  char function_buffer[1024];
  if (task.client->universal._namespace)
  {
    char *ptr= function_buffer;
    memcpy(ptr, gearman_string_value(task.client->universal._namespace), gearman_string_length(task.client->universal._namespace));
    ptr+= gearman_string_length(task.client->universal._namespace);

    memcpy(ptr, gearman_c_str(function), gearman_size(function) +1);
    ptr+= gearman_size(function);

    args[0]= function_buffer;
    args_size[0]= ptr -function_buffer +1;
  }
  else
  {
    args[0]= gearman_c_str(function);
    args_size[0]= gearman_size(function) +1;
  }

  args[1]= task.unique;
  args_size[1]= task.unique_length +1;

  args[2]= gearman_c_str(workload);
  args_size[2]= gearman_size(workload);

  return gearman_packet_create_args(task.client->universal, task.send,
                                    GEARMAN_MAGIC_REQUEST,
                                    command,
                                    args, args_size,
                                    3);
}

gearman_return_t submit(gearman_task_st& task,
                        const gearman_command_t command,
                        const gearman_string_t &function,
                        const gearman_string_t &workload)
{
  return __submit(task, command, function, workload);
}

gearman_return_t submit_background(gearman_task_st& task,
                                   const gearman_command_t command,
                                   const gearman_string_t &function,
                                   const gearman_string_t &workload)
{
  return __submit(task, command, function, workload);
}

gearman_return_t submit_epoch(gearman_task_st& task,
                              const gearman_string_t &function,
                              const gearman_string_t &workload,
                              time_t when)
{
  const void *args[4];
  size_t args_size[4];

  /**
    @todo fix it so that NULL is done by default by the API not by happenstance.
  */
  char function_buffer[1024];
  if (task.client->universal._namespace)
  {
    char *ptr= function_buffer;
    memcpy(ptr, gearman_string_value(task.client->universal._namespace), gearman_string_length(task.client->universal._namespace));
    ptr+= gearman_string_length(task.client->universal._namespace);

    memcpy(ptr, gearman_c_str(function), gearman_size(function) +1);
    ptr+= gearman_size(function);

    args[0]= function_buffer;
    args_size[0]= ptr -function_buffer +1;
  }
  else
  {
    args[0]= gearman_c_str(function);
    args_size[0]= gearman_size(function) +1;
  }

  args[1]= task.unique;
  args_size[1]= task.unique_length +1;

  char time_string[30];
  int length= snprintf(time_string, sizeof(time_string), "%" PRIu64, static_cast<int64_t>(when));
  args[2]= time_string;
  args_size[2]= length +1;

  args[3]= gearman_c_str(workload);
  args_size[3]= gearman_size(workload);

  return gearman_packet_create_args(task.client->universal, task.send,
                                    GEARMAN_MAGIC_REQUEST,
                                    GEARMAN_COMMAND_SUBMIT_JOB_EPOCH,
                                    args, args_size,
                                    4);
}

} // namespace protocol
} // namespace libgearman

<|endoftext|>


<commit_before>/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2012 Artem Pavlenko
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/

#ifndef MAPNIK_JSON_GEOMETRY_GENERATOR_GRAMMAR_HPP
#define MAPNIK_JSON_GEOMETRY_GENERATOR_GRAMMAR_HPP

// mapnik
#include <mapnik/global.hpp>
#include <mapnik/geometry.hpp>
#include <mapnik/util/path_iterator.hpp>
#include <mapnik/util/container_adapter.hpp>
#include <mapnik/vertex.hpp>    // for CommandType::SEG_MOVETO

// boost
#include <boost/tuple/tuple.hpp>
#include <boost/spirit/include/karma.hpp>
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/spirit/include/phoenix_fusion.hpp>
#include <boost/spirit/include/phoenix_function.hpp>
#include <boost/spirit/include/phoenix_statement.hpp>
#include <boost/fusion/include/boost_tuple.hpp>
#include <boost/math/special_functions/trunc.hpp> // for vc++

namespace boost { namespace spirit { namespace traits {

// make gcc and darwin toolsets happy.
template <>
struct is_container<mapnik::geometry_container>
    : mpl::false_
{};

}}}

namespace mapnik { namespace json {

namespace karma = boost::spirit::karma;
namespace phoenix = boost::phoenix;

namespace {

#ifdef BOOST_SPIRIT_USE_PHOENIX_V3
struct get_type
{
    typedef int result_type;
    result_type operator() (geometry_type const& geom) const
    {
        return static_cast<int>(geom.type());
    }
};

struct get_first
{
    typedef geometry_type::value_type const result_type;
    result_type operator() (geometry_type const& geom) const
    {
        geometry_type::value_type coord;
        boost::get<0>(coord) = geom.vertex(0,&boost::get<1>(coord),&boost::get<2>(coord));
        return coord;
    }
};

struct multi_geometry_type
{
    typedef boost::tuple<unsigned,bool>  result_type;
    result_type operator() (geometry_container const& geom) const
    {
        unsigned type = 0u;
        bool collection = false;

        geometry_container::const_iterator itr = geom.begin();
        geometry_container::const_iterator end = geom.end();

        for ( ; itr != end; ++itr)
        {
            if (type != 0u && itr->type() != type)
            {
                collection = true;
                break;
            }
            type = itr->type();
        }
        if (geom.size() > 1) type +=3;
        return boost::tuple<unsigned,bool>(type, collection);
    }
};
#else
struct get_type
{
    template <typename T>
    struct result { typedef int type; };

    int operator() (geometry_type const& geom) const
    {
        return static_cast<int>(geom.type());
    }
};

struct get_first
{
    template <typename T>
    struct result { typedef geometry_type::value_type const type; };

    geometry_type::value_type const operator() (geometry_type const& geom) const
    {
        geometry_type::value_type coord;
        boost::get<0>(coord) = geom.vertex(0,&boost::get<1>(coord),&boost::get<2>(coord));
        return coord;
    }
};

struct multi_geometry_type
{
    template <typename T>
    struct result { typedef boost::tuple<unsigned,bool> type; };

    boost::tuple<unsigned,bool> operator() (geometry_container const& geom) const
    {
        unsigned type = 0u;
        bool collection = false;

        geometry_container::const_iterator itr = geom.begin();
        geometry_container::const_iterator end = geom.end();

        for ( ; itr != end; ++itr)
        {
            if (type != 0u && itr->type() != type)
            {
                collection = true;
                break;
            }
            type = itr->type();
        }
        if (geom.size() > 1) type +=3;
        return boost::tuple<unsigned,bool>(type, collection);
    }
};
#endif

template <typename T>
struct json_coordinate_policy : karma::real_policies<T>
{
    typedef boost::spirit::karma::real_policies<T> base_type;
    static int floatfield(T n) { return base_type::fmtflags::fixed; }

    static unsigned precision(T n)
    {
        if (n == 0.0) return 0;
        using namespace boost::spirit;
        return static_cast<unsigned>(14 - boost::math::trunc(log10(traits::get_absolute_value(n))));
    }

    template <typename OutputIterator>
    static bool dot(OutputIterator& sink, T n, unsigned precision)
    {
        if (n == 0) return true;
        return base_type::dot(sink, n, precision);
    }

    template <typename OutputIterator>
    static bool fraction_part(OutputIterator& sink, T n
                              , unsigned adjprec, unsigned precision)
    {
        if (n == 0) return true;
        return base_type::fraction_part(sink, n, adjprec, precision);
    }
};

}

template <typename OutputIterator>
struct geometry_generator_grammar :
        karma::grammar<OutputIterator, geometry_type const& ()>
{

    geometry_generator_grammar()
        : geometry_generator_grammar::base_type(coordinates)
    {
        using boost::spirit::karma::uint_;
        using boost::spirit::karma::_val;
        using boost::spirit::karma::_1;
        using boost::spirit::karma::lit;
        using boost::spirit::karma::_a;
        using boost::spirit::karma::_r1;
        using boost::spirit::karma::eps;

        coordinates =  point | linestring | polygon
            ;

        point = &uint_(mapnik::Point)[_1 = _type(_val)]
            << point_coord [_1 = _first(_val)]
            ;

        linestring = &uint_(mapnik::LineString)[_1 = _type(_val)]
            << lit('[')
            << coords
            << lit(']')
            ;

        polygon = &uint_(mapnik::Polygon)[_1 = _type(_val)]
            << lit('[')
            << coords2
            << lit("]]")
            ;

        point_coord = &uint_
            << lit('[')
            << coord_type << lit(',') << coord_type
            << lit("]")
            ;

        polygon_coord %= ( &uint_(mapnik::SEG_MOVETO) << eps[_r1 += 1]
                           << karma::string[ if_ (_r1 > 1) [_1 = "],["]
                                             .else_[_1 = '[' ]]
                           |
                           &uint_(mapnik::SEG_LINETO))
            << lit(',')
            << lit('[') << coord_type
            << lit(',')
            << coord_type << lit(']')
            ;

        coords2 %= *polygon_coord(_a)
            ;

        coords = point_coord % lit(',')
            ;

    }
    // rules
    karma::rule<OutputIterator, geometry_type const& ()> coordinates;
    karma::rule<OutputIterator, geometry_type const& ()> point;
    karma::rule<OutputIterator, geometry_type const& ()> linestring;
    karma::rule<OutputIterator, geometry_type const& ()> polygon;

    karma::rule<OutputIterator, geometry_type const& ()> coords;
    karma::rule<OutputIterator, karma::locals<unsigned>, geometry_type const& ()> coords2;
    karma::rule<OutputIterator, geometry_type::value_type ()> point_coord;
    karma::rule<OutputIterator, geometry_type::value_type (unsigned& )> polygon_coord;

    // phoenix functions
    phoenix::function<get_type > _type;
    phoenix::function<get_first> _first;
    //
    karma::real_generator<double, json_coordinate_policy<double> > coord_type;

};

template <typename OutputIterator>
struct multi_geometry_generator_grammar :
        karma::grammar<OutputIterator, karma::locals<boost::tuple<unsigned,bool> >,
                       geometry_container const& ()>
{

    multi_geometry_generator_grammar()
        : multi_geometry_generator_grammar::base_type(start)
    {
        using boost::spirit::karma::lit;
        using boost::spirit::karma::eps;
        using boost::spirit::karma::_val;
        using boost::spirit::karma::_1;
        using boost::spirit::karma::_a;
        using boost::spirit::karma::_r1;

        geometry_types.add
            (mapnik::Point,"\"Point\"")
            (mapnik::LineString,"\"LineString\"")
            (mapnik::Polygon,"\"Polygon\"")
            (mapnik::Point + 3,"\"MultiPoint\"")
            (mapnik::LineString + 3,"\"MultiLineString\"")
            (mapnik::Polygon + 3,"\"MultiPolygon\"")
            ;

        start %= ( eps(phoenix::at_c<1>(_a))[_a = _multi_type(_val)]
                   << lit("{\"type\":\"GeometryCollection\",\"geometries\":[")
                   << geometry_collection << lit("]}")
                   |
                   geometry)
            ;

        geometry_collection = -(geometry2 % lit(','))
            ;

        geometry = (lit("{\"type\":")
                    << geometry_types[_1 = phoenix::at_c<0>(_a)][_a = _multi_type(_val)]
                    << lit(",\"coordinates\":")
                    << karma::string[ phoenix::if_ (phoenix::at_c<0>(_a) > 3) [_1 = '['].else_[_1 = ""]]
                    << coordinates
                    << karma::string[ phoenix::if_ (phoenix::at_c<0>(_a) > 3) [_1 = ']'].else_[_1 = ""]]
                    << lit('}')) | lit("null")
            ;

        geometry2 = lit("{\"type\":")
            << geometry_types[_1 = _a][_a = type_(_val)]
            << lit(",\"coordinates\":")
            << path
            << lit('}')
            ;

        coordinates %= path % lit(',')
            ;

    }
    // rules
    karma::rule<OutputIterator, karma::locals<boost::tuple<unsigned,bool> >,
                geometry_container const&()> start;
    karma::rule<OutputIterator, karma::locals<boost::tuple<unsigned,bool> >,
                geometry_container const&()> geometry_collection;
    karma::rule<OutputIterator, karma::locals<boost::tuple<unsigned,bool> >,
                geometry_container const&()> geometry;
    karma::rule<OutputIterator, karma::locals<unsigned>,
                geometry_type const&()> geometry2;
    karma::rule<OutputIterator, geometry_container const&()> coordinates;
    geometry_generator_grammar<OutputIterator>  path;
    // phoenix
    phoenix::function<multi_geometry_type> _multi_type;
    phoenix::function<get_type > type_;
    // symbols table
    karma::symbols<unsigned, char const*> geometry_types;
};

}}

#endif // MAPNIK_JSON_GEOMETRY_GENERATOR_GRAMMAR_HPP
<commit_msg>fix sign compare warning<commit_after>/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2012 Artem Pavlenko
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/

#ifndef MAPNIK_JSON_GEOMETRY_GENERATOR_GRAMMAR_HPP
#define MAPNIK_JSON_GEOMETRY_GENERATOR_GRAMMAR_HPP

// mapnik
#include <mapnik/global.hpp>
#include <mapnik/geometry.hpp>
#include <mapnik/util/path_iterator.hpp>
#include <mapnik/util/container_adapter.hpp>
#include <mapnik/vertex.hpp>    // for CommandType::SEG_MOVETO

// boost
#include <boost/tuple/tuple.hpp>
#include <boost/spirit/include/karma.hpp>
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/spirit/include/phoenix_fusion.hpp>
#include <boost/spirit/include/phoenix_function.hpp>
#include <boost/spirit/include/phoenix_statement.hpp>
#include <boost/fusion/include/boost_tuple.hpp>
#include <boost/math/special_functions/trunc.hpp> // for vc++

namespace boost { namespace spirit { namespace traits {

// make gcc and darwin toolsets happy.
template <>
struct is_container<mapnik::geometry_container>
    : mpl::false_
{};

}}}

namespace mapnik { namespace json {

namespace karma = boost::spirit::karma;
namespace phoenix = boost::phoenix;

namespace {

#ifdef BOOST_SPIRIT_USE_PHOENIX_V3
struct get_type
{
    typedef int result_type;
    result_type operator() (geometry_type const& geom) const
    {
        return static_cast<int>(geom.type());
    }
};

struct get_first
{
    typedef geometry_type::value_type const result_type;
    result_type operator() (geometry_type const& geom) const
    {
        geometry_type::value_type coord;
        boost::get<0>(coord) = geom.vertex(0,&boost::get<1>(coord),&boost::get<2>(coord));
        return coord;
    }
};

struct multi_geometry_type
{
    typedef boost::tuple<unsigned,bool>  result_type;
    result_type operator() (geometry_container const& geom) const
    {
        unsigned type = 0u;
        bool collection = false;

        geometry_container::const_iterator itr = geom.begin();
        geometry_container::const_iterator end = geom.end();

        for ( ; itr != end; ++itr)
        {
            if (type != 0u && itr->type() != type)
            {
                collection = true;
                break;
            }
            type = itr->type();
        }
        if (geom.size() > 1) type +=3;
        return boost::tuple<unsigned,bool>(type, collection);
    }
};
#else
struct get_type
{
    template <typename T>
    struct result { typedef int type; };

    int operator() (geometry_type const& geom) const
    {
        return static_cast<int>(geom.type());
    }
};

struct get_first
{
    template <typename T>
    struct result { typedef geometry_type::value_type const type; };

    geometry_type::value_type const operator() (geometry_type const& geom) const
    {
        geometry_type::value_type coord;
        boost::get<0>(coord) = geom.vertex(0,&boost::get<1>(coord),&boost::get<2>(coord));
        return coord;
    }
};

struct multi_geometry_type
{
    template <typename T>
    struct result { typedef boost::tuple<unsigned,bool> type; };

    boost::tuple<unsigned,bool> operator() (geometry_container const& geom) const
    {
        unsigned type = 0u;
        bool collection = false;

        geometry_container::const_iterator itr = geom.begin();
        geometry_container::const_iterator end = geom.end();

        for ( ; itr != end; ++itr)
        {
            if (type != 0u && static_cast<unsigned>(itr->type()) != type)
            {
                collection = true;
                break;
            }
            type = itr->type();
        }
        if (geom.size() > 1) type +=3;
        return boost::tuple<unsigned,bool>(type, collection);
    }
};
#endif

template <typename T>
struct json_coordinate_policy : karma::real_policies<T>
{
    typedef boost::spirit::karma::real_policies<T> base_type;
    static int floatfield(T n) { return base_type::fmtflags::fixed; }

    static unsigned precision(T n)
    {
        if (n == 0.0) return 0;
        using namespace boost::spirit;
        return static_cast<unsigned>(14 - boost::math::trunc(log10(traits::get_absolute_value(n))));
    }

    template <typename OutputIterator>
    static bool dot(OutputIterator& sink, T n, unsigned precision)
    {
        if (n == 0) return true;
        return base_type::dot(sink, n, precision);
    }

    template <typename OutputIterator>
    static bool fraction_part(OutputIterator& sink, T n
                              , unsigned adjprec, unsigned precision)
    {
        if (n == 0) return true;
        return base_type::fraction_part(sink, n, adjprec, precision);
    }
};

}

template <typename OutputIterator>
struct geometry_generator_grammar :
        karma::grammar<OutputIterator, geometry_type const& ()>
{

    geometry_generator_grammar()
        : geometry_generator_grammar::base_type(coordinates)
    {
        using boost::spirit::karma::uint_;
        using boost::spirit::karma::_val;
        using boost::spirit::karma::_1;
        using boost::spirit::karma::lit;
        using boost::spirit::karma::_a;
        using boost::spirit::karma::_r1;
        using boost::spirit::karma::eps;

        coordinates =  point | linestring | polygon
            ;

        point = &uint_(mapnik::Point)[_1 = _type(_val)]
            << point_coord [_1 = _first(_val)]
            ;

        linestring = &uint_(mapnik::LineString)[_1 = _type(_val)]
            << lit('[')
            << coords
            << lit(']')
            ;

        polygon = &uint_(mapnik::Polygon)[_1 = _type(_val)]
            << lit('[')
            << coords2
            << lit("]]")
            ;

        point_coord = &uint_
            << lit('[')
            << coord_type << lit(',') << coord_type
            << lit("]")
            ;

        polygon_coord %= ( &uint_(mapnik::SEG_MOVETO) << eps[_r1 += 1]
                           << karma::string[ if_ (_r1 > 1) [_1 = "],["]
                                             .else_[_1 = '[' ]]
                           |
                           &uint_(mapnik::SEG_LINETO))
            << lit(',')
            << lit('[') << coord_type
            << lit(',')
            << coord_type << lit(']')
            ;

        coords2 %= *polygon_coord(_a)
            ;

        coords = point_coord % lit(',')
            ;

    }
    // rules
    karma::rule<OutputIterator, geometry_type const& ()> coordinates;
    karma::rule<OutputIterator, geometry_type const& ()> point;
    karma::rule<OutputIterator, geometry_type const& ()> linestring;
    karma::rule<OutputIterator, geometry_type const& ()> polygon;

    karma::rule<OutputIterator, geometry_type const& ()> coords;
    karma::rule<OutputIterator, karma::locals<unsigned>, geometry_type const& ()> coords2;
    karma::rule<OutputIterator, geometry_type::value_type ()> point_coord;
    karma::rule<OutputIterator, geometry_type::value_type (unsigned& )> polygon_coord;

    // phoenix functions
    phoenix::function<get_type > _type;
    phoenix::function<get_first> _first;
    //
    karma::real_generator<double, json_coordinate_policy<double> > coord_type;

};

template <typename OutputIterator>
struct multi_geometry_generator_grammar :
        karma::grammar<OutputIterator, karma::locals<boost::tuple<unsigned,bool> >,
                       geometry_container const& ()>
{

    multi_geometry_generator_grammar()
        : multi_geometry_generator_grammar::base_type(start)
    {
        using boost::spirit::karma::lit;
        using boost::spirit::karma::eps;
        using boost::spirit::karma::_val;
        using boost::spirit::karma::_1;
        using boost::spirit::karma::_a;
        using boost::spirit::karma::_r1;

        geometry_types.add
            (mapnik::Point,"\"Point\"")
            (mapnik::LineString,"\"LineString\"")
            (mapnik::Polygon,"\"Polygon\"")
            (mapnik::Point + 3,"\"MultiPoint\"")
            (mapnik::LineString + 3,"\"MultiLineString\"")
            (mapnik::Polygon + 3,"\"MultiPolygon\"")
            ;

        start %= ( eps(phoenix::at_c<1>(_a))[_a = _multi_type(_val)]
                   << lit("{\"type\":\"GeometryCollection\",\"geometries\":[")
                   << geometry_collection << lit("]}")
                   |
                   geometry)
            ;

        geometry_collection = -(geometry2 % lit(','))
            ;

        geometry = (lit("{\"type\":")
                    << geometry_types[_1 = phoenix::at_c<0>(_a)][_a = _multi_type(_val)]
                    << lit(",\"coordinates\":")
                    << karma::string[ phoenix::if_ (phoenix::at_c<0>(_a) > 3) [_1 = '['].else_[_1 = ""]]
                    << coordinates
                    << karma::string[ phoenix::if_ (phoenix::at_c<0>(_a) > 3) [_1 = ']'].else_[_1 = ""]]
                    << lit('}')) | lit("null")
            ;

        geometry2 = lit("{\"type\":")
            << geometry_types[_1 = _a][_a = type_(_val)]
            << lit(",\"coordinates\":")
            << path
            << lit('}')
            ;

        coordinates %= path % lit(',')
            ;

    }
    // rules
    karma::rule<OutputIterator, karma::locals<boost::tuple<unsigned,bool> >,
                geometry_container const&()> start;
    karma::rule<OutputIterator, karma::locals<boost::tuple<unsigned,bool> >,
                geometry_container const&()> geometry_collection;
    karma::rule<OutputIterator, karma::locals<boost::tuple<unsigned,bool> >,
                geometry_container const&()> geometry;
    karma::rule<OutputIterator, karma::locals<unsigned>,
                geometry_type const&()> geometry2;
    karma::rule<OutputIterator, geometry_container const&()> coordinates;
    geometry_generator_grammar<OutputIterator>  path;
    // phoenix
    phoenix::function<multi_geometry_type> _multi_type;
    phoenix::function<get_type > type_;
    // symbols table
    karma::symbols<unsigned, char const*> geometry_types;
};

}}

#endif // MAPNIK_JSON_GEOMETRY_GENERATOR_GRAMMAR_HPP
<|endoftext|>


<commit_before>#include <cstring>

namespace accelergy
{
  void invokeAccelergy(std::vector<std::string> inputFiles) {
#ifdef ACCELERGY_PATH
    std::string cmd = std::string(ACCELERGY_PATH) + "/accelergy";
    for (auto inputFile : inputFiles) {
      cmd += " " + inputFile;
    }
    cmd += " -o ./ > ./accelergy.log 2>&1";
    int ret = system(cmd.c_str());
    if (ret) {
      std::cout << "Cannot invoke Accelergy. Do you specify ACCELERGYPATH correctly?" << std::endl;
      exit(0);
    }
#else
    (void) inputFiles;
#endif
    return;
  }
} // namespace accelergy
<commit_msg>[accelergy_interface] better error message<commit_after>#include <cstring>

namespace accelergy
{
  void invokeAccelergy(std::vector<std::string> inputFiles) {
#ifdef ACCELERGY_PATH
    std::string cmd = std::string(ACCELERGY_PATH) + "/accelergy";
    for (auto inputFile : inputFiles) {
      cmd += " " + inputFile;
    }
    cmd += " -o ./ > ./accelergy.log 2>&1";
    int ret = system(cmd.c_str());
    if (ret) {
      std::cout << "Cannot invoke Accelergy. Do you specify ACCELERGYPATH correctly? Or check accelergy.log to see what went wrong" << std::endl;
      exit(0);
    }
#else
    (void) inputFiles;
#endif
    return;
  }
} // namespace accelergy
<|endoftext|>


<commit_before>/*
 *  Copyright (c) 2015, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */
#include "ProxyDestination.h"

#include <folly/Conv.h>
#include <folly/Memory.h>

#include "mcrouter/config-impl.h"
#include "mcrouter/config.h"
#include "mcrouter/lib/fbi/asox_timer.h"
#include "mcrouter/lib/fbi/nstring.h"
#include "mcrouter/lib/fbi/timer.h"
#include "mcrouter/lib/fbi/util.h"
#include "mcrouter/lib/fibers/Fiber.h"
#include "mcrouter/lib/network/AsyncMcClient.h"
#include "mcrouter/pclient.h"
#include "mcrouter/ProxyClientCommon.h"
#include "mcrouter/routes/DestinationRoute.h"
#include "mcrouter/stats.h"
#include "mcrouter/TkoTracker.h"

namespace facebook { namespace memcache { namespace mcrouter {

namespace {

constexpr double kProbeExponentialFactor = 1.5;
constexpr double kProbeJitterMin = 0.05;
constexpr double kProbeJitterMax = 0.5;
constexpr double kProbeJitterDelta = kProbeJitterMax - kProbeJitterMin;

static_assert(kProbeJitterMax >= kProbeJitterMin,
              "ProbeJitterMax should be greater or equal tham ProbeJitterMin");

void on_probe_timer(const asox_timer_t timer, void* arg) {
  ProxyDestination* pdstn = (ProxyDestination*)arg;
  pdstn->on_timer(timer);
}

}  // anonymous namespace

void ProxyDestination::schedule_next_probe() {
  FBI_ASSERT(proxy->magic == proxy_magic);
  FBI_ASSERT(!proxy->opts.disable_tko_tracking);

  int delay_ms = probe_delay_next_ms;
  if (probe_delay_next_ms < 2) {
    // int(1 * 1.5) == 1, so advance it to 2 first
    probe_delay_next_ms = 2;
  } else {
    probe_delay_next_ms *= kProbeExponentialFactor;
  }
  if (probe_delay_next_ms > proxy->opts.probe_delay_max_ms) {
    probe_delay_next_ms = proxy->opts.probe_delay_max_ms;
  }

  // Calculate random jitter
  double r = (double)rand() / (double)RAND_MAX;
  double tmo_jitter_pct = r * kProbeJitterDelta + kProbeJitterMin;
  uint64_t delay_us = (double)delay_ms * 1000 * (1.0 + tmo_jitter_pct);
  FBI_ASSERT(delay_us > 0);

  timeval_t delay;
  delay.tv_sec = (delay_us / 1000000);
  delay.tv_usec = (delay_us % 1000000);

  FBI_ASSERT(probe_timer == nullptr);
  probe_timer = asox_add_timer(proxy->eventBase->getLibeventBase(), delay,
                               on_probe_timer, this);
}

void ProxyDestination::on_timer(const asox_timer_t timer) {
  // This assert checks for use-after-free
  FBI_ASSERT(proxy->magic == proxy_magic);
  FBI_ASSERT(timer == probe_timer);
  asox_remove_timer(timer);
  probe_timer = nullptr;
  if (sending_probes) {
    // Note that the previous probe might still be in flight
    if (!probe_req) {
      auto mutReq = createMcMsgRef();
      mutReq->op = mc_op_version;
      probe_req = folly::make_unique<McRequest>(std::move(mutReq));
      ++probesSent_;
      auto selfPtr = selfPtr_;
      proxy->fiberManager.addTask([selfPtr]() mutable {
        auto pdstn = selfPtr.lock();
        if (pdstn == nullptr) {
          return;
        }
        pdstn->proxy->destinationMap->markAsActive(*pdstn);
        McReply reply = pdstn->client_->send(*pdstn->probe_req,
                                             McOperation<mc_op_version>(),
                                             0,
                                             pdstn->shortestTimeout);
        pdstn->handle_tko(reply, true);
        pdstn->probe_req.reset();
      });
    }
    schedule_next_probe();
  }
}

void ProxyDestination::start_sending_probes() {
  FBI_ASSERT(!sending_probes);
  sending_probes = true;
  probe_delay_next_ms = proxy->opts.probe_delay_initial_ms;
  schedule_next_probe();
}

void ProxyDestination::stop_sending_probes() {
  probesSent_ = 0;
  sending_probes = false;
  if (probe_timer) {
    asox_remove_timer(probe_timer);
    probe_timer = nullptr;
  }
}

void ProxyDestination::unmark_tko(const McReply& reply) {
  FBI_ASSERT(!proxy->opts.disable_tko_tracking);
  shared->tko.recordSuccess(this);
  if (sending_probes) {
    onTkoEvent(TkoLogEvent::UnMarkTko, reply.result());
    stop_sending_probes();
  }
}

void ProxyDestination::handle_tko(const McReply& reply, bool is_probe_req) {
  if (resetting ||
      proxy->opts.disable_tko_tracking) {
    return;
  }

  bool responsible = false;
  if (reply.isError()) {
    if (reply.isHardTkoError()) {
      responsible = shared->tko.recordHardFailure(this);
      if (responsible) {
        onTkoEvent(TkoLogEvent::MarkHardTko, reply.result());
      }
    } else if (reply.isSoftTkoError()) {
      responsible = shared->tko.recordSoftFailure(this);
      if (responsible) {
        onTkoEvent(TkoLogEvent::MarkSoftTko, reply.result());
      }
    }
  } else if (!sending_probes || is_probe_req) {
    /* If we're sending probes, only a probe request should be considered
       successful to avoid outstanding requests from unmarking the box */
    unmark_tko(reply);
  }
  if (responsible) {
    start_sending_probes();
  }
}

void ProxyDestination::onReply(const McReply& reply,
                               DestinationRequestCtx& destreqCtx) {
  FBI_ASSERT(proxy->magic == proxy_magic);

  handle_tko(reply, false);

  stats_.results[reply.result()]++;
  destreqCtx.endTime = nowUs();

  /* For code simplicity we look at latency for making TKO decisions with a
     1 request delay */
  int64_t latency = destreqCtx.endTime - destreqCtx.startTime;
  stats_.avgLatency.insertSample(latency);

  stat_decr(proxy->stats, sum_server_queue_length_stat, 1);
}

void ProxyDestination::on_up() {
  FBI_ASSERT(proxy->magic == proxy_magic);
  FBI_ASSERT(stats_.state != ProxyDestinationState::kUp);

  stat_incr(proxy->stats, server_up_events_stat, 1);
  setState(ProxyDestinationState::kUp);

  VLOG(1) << "server " << pdstnKey << " up (" <<
      stat_get_uint64(proxy->stats, num_servers_up_stat) << " of " <<
      stat_get_uint64(proxy->stats, num_servers_stat) << ")";
}

void ProxyDestination::on_down() {
  FBI_ASSERT(proxy->magic == proxy_magic);

  if (resetting) {
    VLOG(1) << "server " << pdstnKey << " inactive (" <<
        stat_get_uint64(proxy->stats, num_servers_up_stat) << " of " <<
        stat_get_uint64(proxy->stats, num_servers_stat) << ")";
    setState(ProxyDestinationState::kClosed);
  } else {
    VLOG(1) << "server " << pdstnKey << " down (" <<
        stat_get_uint64(proxy->stats, num_servers_up_stat) << " of " <<
        stat_get_uint64(proxy->stats, num_servers_stat) << ")";
    setState(ProxyDestinationState::kDown);
    handle_tko(McReply(mc_res_connect_error),
               /* is_probe_req= */ false);
  }
}

size_t ProxyDestination::getPendingRequestCount() const {
  return client_->getPendingRequestCount();
}

size_t ProxyDestination::getInflightRequestCount() const {
  return client_->getInflightRequestCount();
}

std::pair<uint64_t, uint64_t> ProxyDestination::getBatchingStat() const {
  return client_->getBatchingStat();
}

std::shared_ptr<ProxyDestination> ProxyDestination::create(
    proxy_t* proxy,
    const ProxyClientCommon& ro,
    std::string pdstnKey) {

  auto ptr = std::shared_ptr<ProxyDestination>(
    new ProxyDestination(proxy, ro, std::move(pdstnKey)));
  ptr->selfPtr_ = ptr;
  ptr->client_ = folly::make_unique<DestinationMcClient>(*ptr);
  return ptr;
}

ProxyDestination::~ProxyDestination() {
  shared->removeDestination(this);
  if (proxy->destinationMap) {
    proxy->destinationMap->removeDestination(*this);
  }

  // should not call 'handle_tko'
  resetting = 1;
  client_.reset();

  if (sending_probes) {
    onTkoEvent(TkoLogEvent::RemoveFromConfig, mc_res_ok);
    stop_sending_probes();
  }

  magic = kDeadBeef;
}

ProxyDestination::ProxyDestination(proxy_t* proxy_,
                                   const ProxyClientCommon& ro_,
                                   std::string pdstnKey_)
  : proxy(proxy_),
    accessPoint(ro_.ap),
    destinationKey(ro_.destination_key),
    shortestTimeout(ro_.server_timeout),
    pdstnKey(std::move(pdstnKey_)),
    proxy_magic(proxy->magic),
    use_ssl(ro_.useSsl),
    qos(ro_.qos),
    stats_(proxy_->opts) {

  static uint64_t next_magic = 0x12345678900000LL;
  magic = __sync_fetch_and_add(&next_magic, 1);
  stat_incr(proxy->stats, num_servers_new_stat, 1);
}

ProxyDestinationState ProxyDestination::state() const {
  if (shared->tko.isTko()) {
    return ProxyDestinationState::kTko;
  }
  return stats_.state;
}

const ProxyDestinationStats& ProxyDestination::stats() const {
  return stats_;
}

bool ProxyDestination::may_send() {
  FBI_ASSERT(proxy->magic == proxy_magic);

  return !shared->tko.isTko();
}

void ProxyDestination::resetInactive() {
  FBI_ASSERT(proxy->magic == proxy_magic);

  resetting = 1;
  client_->resetInactive();
  resetting = 0;
}

void ProxyDestination::onTkoEvent(TkoLogEvent event, mc_res_t result) const {
  auto logUtil = [this, result](folly::StringPiece eventStr) {
    VLOG(1) << shared->key << " " << eventStr << ". Total hard TKOs: "
            << shared->tko.globalTkos().hardTkos << "; soft TKOs: "
            << shared->tko.globalTkos().softTkos << ". Reply: "
            << mc_res_to_string(result);
  };

  switch (event) {
    case TkoLogEvent::MarkHardTko:
      logUtil("marked hard TKO");
      break;
    case TkoLogEvent::MarkSoftTko:
      logUtil("marked soft TKO");
      break;
    case TkoLogEvent::UnMarkTko:
      logUtil("unmarked TKO");
      break;
    case TkoLogEvent::RemoveFromConfig:
      logUtil("was TKO, removed from config");
      break;
  }

  TkoLog tkoLog(accessPoint, shared->tko.globalTkos());
  tkoLog.event = event;
  tkoLog.isHardTko = shared->tko.isHardTko();
  tkoLog.isSoftTko = shared->tko.isSoftTko();
  tkoLog.avgLatency = stats_.avgLatency.value();
  tkoLog.probesSent = probesSent_;
  tkoLog.result = result;

  logTkoEvent(proxy, tkoLog);
}

void ProxyDestination::setState(ProxyDestinationState new_st) {
  auto old_st = stats_.state;

  if (old_st != new_st) {
    auto getStatName = [](ProxyDestinationState st) {
      switch (st) {
        case ProxyDestinationState::kNew:
          return num_servers_new_stat;
        case ProxyDestinationState::kUp:
          return num_servers_up_stat;
        case ProxyDestinationState::kClosed:
          return num_servers_closed_stat;
        case ProxyDestinationState::kDown:
          return num_servers_down_stat;
        default:
          CHECK(false);
          return num_stats; // shouldnt reach here
      }
    };

    auto old_name = getStatName(old_st);
    auto new_name = getStatName(new_st);
    stat_decr(proxy->stats, old_name, 1);
    stat_incr(proxy->stats, new_name, 1);
    stats_.state = new_st;
  }
}

ProxyDestinationStats::ProxyDestinationStats(const McrouterOptions& opts)
  : avgLatency(1.0 / opts.latency_window_size) {
}

void ProxyDestination::updateShortestTimeout(
    std::chrono::milliseconds timeout) {
  if (!timeout.count()) {
    return;
  }
  if (shortestTimeout.count() == 0 || shortestTimeout > timeout) {
    shortestTimeout = timeout;
  }
  client_->updateWriteTimeout();
}

}}}  // facebook::memcache::mcrouter
<commit_msg>Fix mcrouter_startup<commit_after>/*
 *  Copyright (c) 2015, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */
#include "ProxyDestination.h"

#include <folly/Conv.h>
#include <folly/Memory.h>

#include "mcrouter/config-impl.h"
#include "mcrouter/config.h"
#include "mcrouter/lib/fbi/asox_timer.h"
#include "mcrouter/lib/fbi/nstring.h"
#include "mcrouter/lib/fbi/timer.h"
#include "mcrouter/lib/fbi/util.h"
#include "mcrouter/lib/fibers/Fiber.h"
#include "mcrouter/lib/network/AsyncMcClient.h"
#include "mcrouter/pclient.h"
#include "mcrouter/ProxyClientCommon.h"
#include "mcrouter/routes/DestinationRoute.h"
#include "mcrouter/stats.h"
#include "mcrouter/TkoTracker.h"

namespace facebook { namespace memcache { namespace mcrouter {

namespace {

constexpr double kProbeExponentialFactor = 1.5;
constexpr double kProbeJitterMin = 0.05;
constexpr double kProbeJitterMax = 0.5;
constexpr double kProbeJitterDelta = kProbeJitterMax - kProbeJitterMin;

static_assert(kProbeJitterMax >= kProbeJitterMin,
              "ProbeJitterMax should be greater or equal tham ProbeJitterMin");

void on_probe_timer(const asox_timer_t timer, void* arg) {
  ProxyDestination* pdstn = (ProxyDestination*)arg;
  pdstn->on_timer(timer);
}

}  // anonymous namespace

void ProxyDestination::schedule_next_probe() {
  FBI_ASSERT(proxy->magic == proxy_magic);
  FBI_ASSERT(!proxy->opts.disable_tko_tracking);

  int delay_ms = probe_delay_next_ms;
  if (probe_delay_next_ms < 2) {
    // int(1 * 1.5) == 1, so advance it to 2 first
    probe_delay_next_ms = 2;
  } else {
    probe_delay_next_ms *= kProbeExponentialFactor;
  }
  if (probe_delay_next_ms > proxy->opts.probe_delay_max_ms) {
    probe_delay_next_ms = proxy->opts.probe_delay_max_ms;
  }

  // Calculate random jitter
  double r = (double)rand() / (double)RAND_MAX;
  double tmo_jitter_pct = r * kProbeJitterDelta + kProbeJitterMin;
  uint64_t delay_us = (double)delay_ms * 1000 * (1.0 + tmo_jitter_pct);
  FBI_ASSERT(delay_us > 0);

  timeval_t delay;
  delay.tv_sec = (delay_us / 1000000);
  delay.tv_usec = (delay_us % 1000000);

  FBI_ASSERT(probe_timer == nullptr);
  probe_timer = asox_add_timer(proxy->eventBase->getLibeventBase(), delay,
                               on_probe_timer, this);
}

void ProxyDestination::on_timer(const asox_timer_t timer) {
  // This assert checks for use-after-free
  FBI_ASSERT(proxy->magic == proxy_magic);
  FBI_ASSERT(timer == probe_timer);
  asox_remove_timer(timer);
  probe_timer = nullptr;
  if (sending_probes) {
    // Note that the previous probe might still be in flight
    if (!probe_req) {
      auto mutReq = createMcMsgRef();
      mutReq->op = mc_op_version;
      probe_req = folly::make_unique<McRequest>(std::move(mutReq));
      ++probesSent_;
      auto selfPtr = selfPtr_;
      proxy->fiberManager.addTask([selfPtr]() mutable {
        auto pdstn = selfPtr.lock();
        if (pdstn == nullptr) {
          return;
        }
        pdstn->proxy->destinationMap->markAsActive(*pdstn);
        McReply reply = pdstn->client_->send(*pdstn->probe_req,
                                             McOperation<mc_op_version>(),
                                             0,
                                             pdstn->shortestTimeout);
        pdstn->handle_tko(reply, true);
        pdstn->probe_req.reset();
      });
    }
    schedule_next_probe();
  }
}

void ProxyDestination::start_sending_probes() {
  FBI_ASSERT(!sending_probes);
  sending_probes = true;
  probe_delay_next_ms = proxy->opts.probe_delay_initial_ms;
  schedule_next_probe();
}

void ProxyDestination::stop_sending_probes() {
  probesSent_ = 0;
  sending_probes = false;
  if (probe_timer) {
    asox_remove_timer(probe_timer);
    probe_timer = nullptr;
  }
}

void ProxyDestination::unmark_tko(const McReply& reply) {
  FBI_ASSERT(!proxy->opts.disable_tko_tracking);
  shared->tko.recordSuccess(this);
  if (sending_probes) {
    onTkoEvent(TkoLogEvent::UnMarkTko, reply.result());
    stop_sending_probes();
  }
}

void ProxyDestination::handle_tko(const McReply& reply, bool is_probe_req) {
  if (resetting ||
      proxy->opts.disable_tko_tracking) {
    return;
  }

  bool responsible = false;
  if (reply.isError()) {
    if (reply.isHardTkoError()) {
      responsible = shared->tko.recordHardFailure(this);
      if (responsible) {
        onTkoEvent(TkoLogEvent::MarkHardTko, reply.result());
      }
    } else if (reply.isSoftTkoError()) {
      responsible = shared->tko.recordSoftFailure(this);
      if (responsible) {
        onTkoEvent(TkoLogEvent::MarkSoftTko, reply.result());
      }
    }
  } else if (!sending_probes || is_probe_req) {
    /* If we're sending probes, only a probe request should be considered
       successful to avoid outstanding requests from unmarking the box */
    unmark_tko(reply);
  }
  if (responsible) {
    start_sending_probes();
  }
}

void ProxyDestination::onReply(const McReply& reply,
                               DestinationRequestCtx& destreqCtx) {
  FBI_ASSERT(proxy->magic == proxy_magic);

  handle_tko(reply, false);

  stats_.results[reply.result()]++;
  destreqCtx.endTime = nowUs();

  /* For code simplicity we look at latency for making TKO decisions with a
     1 request delay */
  int64_t latency = destreqCtx.endTime - destreqCtx.startTime;
  stats_.avgLatency.insertSample(latency);

  stat_decr(proxy->stats, sum_server_queue_length_stat, 1);
}

void ProxyDestination::on_up() {
  FBI_ASSERT(proxy->magic == proxy_magic);
  FBI_ASSERT(stats_.state != ProxyDestinationState::kUp);

  stat_incr(proxy->stats, server_up_events_stat, 1);
  setState(ProxyDestinationState::kUp);

  VLOG(1) << "server " << pdstnKey << " up (" <<
      stat_get_uint64(proxy->stats, num_servers_up_stat) << " of " <<
      stat_get_uint64(proxy->stats, num_servers_stat) << ")";
}

void ProxyDestination::on_down() {
  FBI_ASSERT(proxy->magic == proxy_magic);

  if (resetting) {
    VLOG(1) << "server " << pdstnKey << " inactive (" <<
        stat_get_uint64(proxy->stats, num_servers_up_stat) << " of " <<
        stat_get_uint64(proxy->stats, num_servers_stat) << ")";
    setState(ProxyDestinationState::kClosed);
  } else {
    VLOG(1) << "server " << pdstnKey << " down (" <<
        stat_get_uint64(proxy->stats, num_servers_up_stat) << " of " <<
        stat_get_uint64(proxy->stats, num_servers_stat) << ")";
    setState(ProxyDestinationState::kDown);
    handle_tko(McReply(mc_res_connect_error),
               /* is_probe_req= */ false);
  }
}

size_t ProxyDestination::getPendingRequestCount() const {
  return client_->getPendingRequestCount();
}

size_t ProxyDestination::getInflightRequestCount() const {
  return client_->getInflightRequestCount();
}

std::pair<uint64_t, uint64_t> ProxyDestination::getBatchingStat() const {
  return client_->getBatchingStat();
}

std::shared_ptr<ProxyDestination> ProxyDestination::create(
    proxy_t* proxy,
    const ProxyClientCommon& ro,
    std::string pdstnKey) {

  auto ptr = std::shared_ptr<ProxyDestination>(
    new ProxyDestination(proxy, ro, std::move(pdstnKey)));
  ptr->selfPtr_ = ptr;
  ptr->client_ = folly::make_unique<DestinationMcClient>(*ptr);
  return ptr;
}

ProxyDestination::~ProxyDestination() {
  shared->removeDestination(this);
  if (proxy->destinationMap) {
    proxy->destinationMap->removeDestination(*this);
  }

  // should not call 'handle_tko'
  resetting = 1;
  client_.reset();

  if (sending_probes) {
    onTkoEvent(TkoLogEvent::RemoveFromConfig, mc_res_ok);
    stop_sending_probes();
  }

  magic = kDeadBeef;
}

ProxyDestination::ProxyDestination(proxy_t* proxy_,
                                   const ProxyClientCommon& ro_,
                                   std::string pdstnKey_)
  : proxy(proxy_),
    accessPoint(ro_.ap),
    destinationKey(ro_.destination_key),
    shortestTimeout(ro_.server_timeout),
    pdstnKey(std::move(pdstnKey_)),
    proxy_magic(proxy->magic),
    use_ssl(ro_.useSsl),
    qos(ro_.qos),
    stats_(proxy_->opts) {

  static uint64_t next_magic = 0x12345678900000LL;
  magic = __sync_fetch_and_add(&next_magic, 1);
  stat_incr(proxy->stats, num_servers_new_stat, 1);
}

ProxyDestinationState ProxyDestination::state() const {
  if (shared->tko.isTko()) {
    return ProxyDestinationState::kTko;
  }
  return stats_.state;
}

const ProxyDestinationStats& ProxyDestination::stats() const {
  return stats_;
}

bool ProxyDestination::may_send() {
  FBI_ASSERT(proxy->magic == proxy_magic);

  return !shared->tko.isTko();
}

void ProxyDestination::resetInactive() {
  FBI_ASSERT(proxy->magic == proxy_magic);

  resetting = 1;
  client_->resetInactive();
  resetting = 0;
}

void ProxyDestination::onTkoEvent(TkoLogEvent event, mc_res_t result) const {
  auto logUtil = [this, result](folly::StringPiece eventStr) {
    VLOG(1) << shared->key << " " << eventStr << ". Total hard TKOs: "
            << shared->tko.globalTkos().hardTkos << "; soft TKOs: "
            << shared->tko.globalTkos().softTkos << ". Reply: "
            << mc_res_to_string(result);
  };

  switch (event) {
    case TkoLogEvent::MarkHardTko:
      logUtil("marked hard TKO");
      break;
    case TkoLogEvent::MarkSoftTko:
      logUtil("marked soft TKO");
      break;
    case TkoLogEvent::UnMarkTko:
      logUtil("unmarked TKO");
      break;
    case TkoLogEvent::RemoveFromConfig:
      logUtil("was TKO, removed from config");
      break;
  }

  TkoLog tkoLog(accessPoint, shared->tko.globalTkos());
  tkoLog.event = event;
  tkoLog.isHardTko = shared->tko.isHardTko();
  tkoLog.isSoftTko = shared->tko.isSoftTko();
  tkoLog.avgLatency = stats_.avgLatency.value();
  tkoLog.probesSent = probesSent_;
  tkoLog.result = result;

  logTkoEvent(proxy, tkoLog);
}

void ProxyDestination::setState(ProxyDestinationState new_st) {
  auto old_st = stats_.state;

  if (old_st != new_st) {
    auto getStatName = [](ProxyDestinationState st) {
      switch (st) {
        case ProxyDestinationState::kNew:
          return num_servers_new_stat;
        case ProxyDestinationState::kUp:
          return num_servers_up_stat;
        case ProxyDestinationState::kClosed:
          return num_servers_closed_stat;
        case ProxyDestinationState::kDown:
          return num_servers_down_stat;
        default:
          CHECK(false);
          return num_stats; // shouldnt reach here
      }
    };

    auto old_name = getStatName(old_st);
    auto new_name = getStatName(new_st);
    stat_decr(proxy->stats, old_name, 1);
    stat_incr(proxy->stats, new_name, 1);
    stats_.state = new_st;
  }
}

ProxyDestinationStats::ProxyDestinationStats(const McrouterOptions& opts)
  : avgLatency(1.0 / opts.latency_window_size) {
}

void ProxyDestination::updateShortestTimeout(
    std::chrono::milliseconds timeout) {
  if (!timeout.count()) {
    return;
  }
  if (shortestTimeout.count() == 0 || shortestTimeout > timeout) {
    shortestTimeout = timeout;
    client_->updateWriteTimeout();
  }
}

}}}  // facebook::memcache::mcrouter
<|endoftext|>


<commit_before>//
// Log - simple message logging to a file.
//
// Copyright (c) 2002-2005 Virtual Terrain Project
// Free for all uses, see license.txt for details.
//

#include "cpl_error.h"	// From GDAL/include
#include "vtString.h"
#include "vtLog.h"
#include <stdarg.h>

#ifndef __DARWIN_OSX__
#include <wchar.h>		// for fputws()
#endif

#ifdef _MSC_VER
#include <windows.h>	// for OutputDebugString, unfortunately
#endif

vtLog g_Log;

vtLog::vtLog()
{
	m_log = NULL;
}

vtLog::~vtLog()
{
	if (m_log)
	{
		fclose(m_log);
		m_log = NULL;
	}
}

void vtLog::StartLog(const char *fname)
{
	m_log = fopen(fname, "wb");
}

void vtLog::Log(const char *msg)
{
	if (m_log)
	{
		fputs(msg, m_log);
		fflush(m_log);
	}
#ifdef _MSC_VER
	OutputDebugStringA(msg);
#endif
	// also send to the console, for those console-mode developers!
	fputs(msg, stdout);
}

void vtLog::Log(char ch)
{
	if (m_log)
	{
		fputc(ch, m_log);
		fflush(m_log);
	}
#ifdef _MSC_VER
	char str[2];
	str[0] = ch;
	str[1] = 0;
	OutputDebugStringA(str);
#endif
	// also send to the console, for those console-mode developers!
	fputc(ch, stdout);
}

void vtLog::Printf(const char *pFormat, ...)
{
	va_list va;
	va_start(va, pFormat);

	char ach[2048];
	vsprintf(ach, pFormat, va);

	Log(ach);
}

#if SUPPORT_WSTRING

void vtLog::Log(const wchar_t *msg)
{
	if (m_log)
	{
		// it is not so useful to write wide characters to the file, which
		// otherwise contains 8-bit text, so convert back first
//		fputws(msg, m_log);
		wstring2 str = msg;
		fputs(str.eb_str(), m_log);
		fflush(m_log);
	}

#ifdef _MSC_VER
	OutputDebugStringW(msg);
#endif
}

void vtLog::Printf(const wchar_t *pFormat, ...)
{
	va_list va;
	va_start(va, pFormat);

	// Use wide characters
	wchar_t ach[2048];

#if defined(_MSC_VER) && _MSC_VER < 1300
	vswprintf(ach, pFormat, va);
#else
	// on MSVC7.x and non-MSVC platforms this takes 4 arguments (safer)
	vswprintf(ach, 2048, pFormat, va);
#endif

	Log(ach);
}

#endif // SUPPORT_WSTRING

<commit_msg>also sent wide strings to stdout, for Linux developers<commit_after>//
// Log - simple message logging to a file.
//
// Copyright (c) 2002-2005 Virtual Terrain Project
// Free for all uses, see license.txt for details.
//

#include "cpl_error.h"	// From GDAL/include
#include "vtString.h"
#include "vtLog.h"
#include <stdarg.h>

#ifndef __DARWIN_OSX__
#include <wchar.h>		// for fputws()
#endif

#ifdef _MSC_VER
#include <windows.h>	// for OutputDebugString, unfortunately
#endif

vtLog g_Log;

vtLog::vtLog()
{
	m_log = NULL;
}

vtLog::~vtLog()
{
	if (m_log)
	{
		fclose(m_log);
		m_log = NULL;
	}
}

void vtLog::StartLog(const char *fname)
{
	m_log = fopen(fname, "wb");
}

void vtLog::Log(const char *msg)
{
	if (m_log)
	{
		fputs(msg, m_log);
		fflush(m_log);
	}
#ifdef _MSC_VER
	OutputDebugStringA(msg);
#endif
	// also send to the console, for those console-mode developers!
	fputs(msg, stdout);
}

void vtLog::Log(char ch)
{
	if (m_log)
	{
		fputc(ch, m_log);
		fflush(m_log);
	}
#ifdef _MSC_VER
	char str[2];
	str[0] = ch;
	str[1] = 0;
	OutputDebugStringA(str);
#endif
	// also send to the console, for those console-mode developers!
	fputc(ch, stdout);
}

void vtLog::Printf(const char *pFormat, ...)
{
	va_list va;
	va_start(va, pFormat);

	char ach[2048];
	vsprintf(ach, pFormat, va);

	Log(ach);
}

#if SUPPORT_WSTRING

void vtLog::Log(const wchar_t *msg)
{
	wstring2 str = msg;
	const char *ebstr = str.eb_str();
	if (m_log)
	{
		// it is not so useful to write wide characters to the file, which
		// otherwise contains 8-bit text, so convert back first
//		fputws(msg, m_log);
		fputs(ebstr, m_log);
		fflush(m_log);
	}
	// also send to the console, for those console-mode developers!
	fputs(ebstr, stdout);

#ifdef _MSC_VER
	OutputDebugStringW(msg);
#endif
}

void vtLog::Printf(const wchar_t *pFormat, ...)
{
	va_list va;
	va_start(va, pFormat);

	// Use wide characters
	wchar_t ach[2048];

#if defined(_MSC_VER) && _MSC_VER < 1300
	vswprintf(ach, pFormat, va);
#else
	// on MSVC7.x and non-MSVC platforms this takes 4 arguments (safer)
	vswprintf(ach, 2048, pFormat, va);
#endif

	Log(ach);
}

#endif // SUPPORT_WSTRING

<|endoftext|>