<commit_before>// Copyright 2018 Google LLC
// Copyright 2018-present Open Networking Foundation
// SPDX-License-Identifier: Apache-2.0

// Implementation of p4_utils functions.

#include "stratum/hal/lib/p4/utils.h"

#include <arpa/inet.h>

#include "absl/strings/str_format.h"
#include "absl/strings/substitute.h"
#include "google/rpc/code.pb.h"
#include "google/rpc/status.pb.h"
#include "p4/config/v1/p4info.pb.h"
#include "stratum/glue/gtl/map_util.h"
#include "stratum/lib/macros.h"
#include "stratum/lib/utils.h"
#include "stratum/public/lib/error.h"

namespace stratum {
namespace hal {

// Decodes a P4 object ID into a human-readable form.  The high-order byte of
// the 32-bit ID is a resource type, as specified by the P4Ids::Prefix enum.
std::string PrintP4ObjectID(int object_id) {
  int base_id = object_id & 0xffffff;
  ::p4::config::v1::P4Ids::Prefix resource_type =
      static_cast<::p4::config::v1::P4Ids::Prefix>((object_id >> 24) & 0xff);
  std::string resource_name =
      ::p4::config::v1::P4Ids::Prefix_Name(resource_type);
  if (resource_name.empty()) resource_name = "INVALID";
  return absl::StrFormat("%s/0x%x (0x%x)", resource_name.c_str(), base_id,
                         object_id);
}

// This unnamed namespace hides a function that forms a status string to refer
// to a P4 object.
namespace {

std::string AddP4ObjectReferenceString(const std::string& log_p4_object) {
  std::string referenced_object;
  if (!log_p4_object.empty()) {
    referenced_object =
        absl::Substitute(" referenced by P4 object $0", log_p4_object.c_str());
  }
  return referenced_object;
}

}  // namespace

::util::StatusOr<const P4TableMapValue*> GetTableMapValueWithDescriptorCase(
    const P4PipelineConfig& p4_pipeline_config,
    const std::string& table_map_key,
    P4TableMapValue::DescriptorCase descriptor_case,
    const std::string& log_p4_object) {
  auto* map_value =
      gtl::FindOrNull(p4_pipeline_config.table_map(), table_map_key);
  if (map_value != nullptr) {
    if (map_value->descriptor_case() != descriptor_case) {
      return MAKE_ERROR(ERR_INTERNAL)
             << "P4PipelineConfig descriptor for " << table_map_key
             << AddP4ObjectReferenceString(log_p4_object)
             << " does not have the expected descriptor case: "
             << map_value->ShortDebugString();
    }
  } else {
    return MAKE_ERROR(ERR_INTERNAL)
           << "P4PipelineConfig table map has no descriptor for "
           << table_map_key << AddP4ObjectReferenceString(log_p4_object);
  }

  return map_value;
}

std::string Uint64ToByteStream(uint64 val) {
  uint64 tmp = (htonl(1) == 1)
                   ? val
                   : (static_cast<uint64>(htonl(val)) << 32) | htonl(val >> 32);
  std::string bytes = "";
  bytes.assign(reinterpret_cast<char*>(&tmp), sizeof(uint64));
  // Strip leading zeroes.
  while (bytes.size() > 1 && bytes[0] == '\x00') {
    bytes = bytes.substr(1);
  }
  return bytes;
}

std::string Uint32ToByteStream(uint32 val) {
  uint32 tmp = htonl(val);
  std::string bytes = "";
  bytes.assign(reinterpret_cast<char*>(&tmp), sizeof(uint32));
  // Strip leading zeroes.
  while (bytes.size() > 1 && bytes[0] == '\x00') {
    bytes = bytes.substr(1);
  }
  return bytes;
}

std::string P4RuntimeGrpcStatusToString(const ::grpc::Status& status) {
  std::stringstream ss;
  if (!status.error_details().empty()) {
    ss << "(overall error code: "
       << ::google::rpc::Code_Name(ToGoogleRpcCode(status.error_code()))
       << ", overall error message: "
       << (status.error_message().empty() ? "None" : status.error_message())
       << "). Error details: ";
    ::google::rpc::Status details;
    if (!details.ParseFromString(status.error_details())) {
      ss << "Failed to parse ::google::rpc::Status from GRPC status details.";
    } else {
      for (int i = 0; i < details.details_size(); ++i) {
        ::p4::v1::Error detail;
        if (details.details(i).UnpackTo(&detail)) {
          ss << "\n(error #" << i + 1 << ": error code: "
             << ::google::rpc::Code_Name(ToGoogleRpcCode(detail.code()))
             << ", error message: "
             << (detail.message().empty() ? "None" : detail.message()) << ") ";
        }
      }
    }
  } else {
    ss << "(error code: "
       << ::google::rpc::Code_Name(ToGoogleRpcCode(status.error_code()))
       << ", error message: "
       << (status.error_message().empty() ? "None" : status.error_message())
       << ").";
  }

  return ss.str();
}

}  // namespace hal
}  // namespace stratum
<commit_msg>Fix a unreachable code error in P4 utils with clang on MacOS (#569)<commit_after>// Copyright 2018 Google LLC
// Copyright 2018-present Open Networking Foundation
// SPDX-License-Identifier: Apache-2.0

// Implementation of p4_utils functions.

#include "stratum/hal/lib/p4/utils.h"

#include <arpa/inet.h>

#include "absl/strings/str_format.h"
#include "absl/strings/substitute.h"
#include "google/rpc/code.pb.h"
#include "google/rpc/status.pb.h"
#include "p4/config/v1/p4info.pb.h"
#include "stratum/glue/gtl/map_util.h"
#include "stratum/lib/macros.h"
#include "stratum/lib/utils.h"
#include "stratum/public/lib/error.h"

namespace stratum {
namespace hal {

// Decodes a P4 object ID into a human-readable form.  The high-order byte of
// the 32-bit ID is a resource type, as specified by the P4Ids::Prefix enum.
std::string PrintP4ObjectID(int object_id) {
  int base_id = object_id & 0xffffff;
  ::p4::config::v1::P4Ids::Prefix resource_type =
      static_cast<::p4::config::v1::P4Ids::Prefix>((object_id >> 24) & 0xff);
  std::string resource_name =
      ::p4::config::v1::P4Ids::Prefix_Name(resource_type);
  if (resource_name.empty()) resource_name = "INVALID";
  return absl::StrFormat("%s/0x%x (0x%x)", resource_name.c_str(), base_id,
                         object_id);
}

// This unnamed namespace hides a function that forms a status string to refer
// to a P4 object.
namespace {

std::string AddP4ObjectReferenceString(const std::string& log_p4_object) {
  std::string referenced_object;
  if (!log_p4_object.empty()) {
    referenced_object =
        absl::Substitute(" referenced by P4 object $0", log_p4_object.c_str());
  }
  return referenced_object;
}

}  // namespace

::util::StatusOr<const P4TableMapValue*> GetTableMapValueWithDescriptorCase(
    const P4PipelineConfig& p4_pipeline_config,
    const std::string& table_map_key,
    P4TableMapValue::DescriptorCase descriptor_case,
    const std::string& log_p4_object) {
  auto* map_value =
      gtl::FindOrNull(p4_pipeline_config.table_map(), table_map_key);
  if (map_value != nullptr) {
    if (map_value->descriptor_case() != descriptor_case) {
      return MAKE_ERROR(ERR_INTERNAL)
             << "P4PipelineConfig descriptor for " << table_map_key
             << AddP4ObjectReferenceString(log_p4_object)
             << " does not have the expected descriptor case: "
             << map_value->ShortDebugString();
    }
  } else {
    return MAKE_ERROR(ERR_INTERNAL)
           << "P4PipelineConfig table map has no descriptor for "
           << table_map_key << AddP4ObjectReferenceString(log_p4_object);
  }

  return map_value;
}

std::string Uint64ToByteStream(uint64 val) {
  uint64 tmp = (htonl(1) == (1))
                   ? val
                   : (static_cast<uint64>(htonl(val)) << 32) | htonl(val >> 32);
  std::string bytes = "";
  bytes.assign(reinterpret_cast<char*>(&tmp), sizeof(uint64));
  // Strip leading zeroes.
  while (bytes.size() > 1 && bytes[0] == '\x00') {
    bytes = bytes.substr(1);
  }
  return bytes;
}

std::string Uint32ToByteStream(uint32 val) {
  uint32 tmp = htonl(val);
  std::string bytes = "";
  bytes.assign(reinterpret_cast<char*>(&tmp), sizeof(uint32));
  // Strip leading zeroes.
  while (bytes.size() > 1 && bytes[0] == '\x00') {
    bytes = bytes.substr(1);
  }
  return bytes;
}

std::string P4RuntimeGrpcStatusToString(const ::grpc::Status& status) {
  std::stringstream ss;
  if (!status.error_details().empty()) {
    ss << "(overall error code: "
       << ::google::rpc::Code_Name(ToGoogleRpcCode(status.error_code()))
       << ", overall error message: "
       << (status.error_message().empty() ? "None" : status.error_message())
       << "). Error details: ";
    ::google::rpc::Status details;
    if (!details.ParseFromString(status.error_details())) {
      ss << "Failed to parse ::google::rpc::Status from GRPC status details.";
    } else {
      for (int i = 0; i < details.details_size(); ++i) {
        ::p4::v1::Error detail;
        if (details.details(i).UnpackTo(&detail)) {
          ss << "\n(error #" << i + 1 << ": error code: "
             << ::google::rpc::Code_Name(ToGoogleRpcCode(detail.code()))
             << ", error message: "
             << (detail.message().empty() ? "None" : detail.message()) << ") ";
        }
      }
    }
  } else {
    ss << "(error code: "
       << ::google::rpc::Code_Name(ToGoogleRpcCode(status.error_code()))
       << ", error message: "
       << (status.error_message().empty() ? "None" : status.error_message())
       << ").";
  }

  return ss.str();
}

}  // namespace hal
}  // namespace stratum
<|endoftext|>


<commit_before>// Smooth - C++ framework for writing applications based on Espressif's ESP-IDF.
// Copyright (C) 2017 Per Malmberg (https://github.com/PerMalmberg)
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

#include <smooth/application/network/http/websocket/WebsocketProtocol.h>
#include <smooth/core/util/string_util.h>
#include <smooth/application/network/http/regular/HTTPHeaderDef.h>
#include <smooth/application/network/http/regular/RegularHTTPProtocol.h>
#include <smooth/application/network/http/regular/responses/ErrorResponse.h>
#include <smooth/core/network/util.h>

namespace smooth::application::network::http::websocket
{
    using namespace smooth::core;
    using namespace smooth::core::network;
    using namespace smooth::application::network::http::regular;

    int WebsocketProtocol::get_wanted_amount(HTTPPacket& packet)
    {
        int len;
        if (state == State::Header)
        {
            len = 2;
        }
        else if (state == State::ExtendedPayloadLength_2)
        {
            len = 2;
        }
        else if (state == State::ExtendedPayloadLength_8)
        {
            len = 8;
        }
        else if (state == State::MaskingKey)
        {
            len = 4;
        }
        else
        {
            // Websockets can use up to 64 bits (well, 63 since MSB MUST always be 0,
            // see https://tools.ietf.org/html/rfc6455#section-5.2) to specify the size.
            // Since we can't handle that large data buffers, we split it into smaller parts based on content_chunk_size

            // First get the maximum number of bytes to read, this limits the value to fit in the type of
            // content_chunk_size, making the following cast safe too.
            auto size = std::min(payload_length - received_payload,
                                 static_cast<decltype(payload_length)>(content_chunk_size));
            len = static_cast<decltype(content_chunk_size)>(size);
            // Actual space needed is what whe already have, plus what we want to read.
            auto space_needed = static_cast<decltype(content_chunk_size)>(received_payload_in_current_package) + len;
            packet.ensure_room(space_needed);
        }

        return len;
    }

    void WebsocketProtocol::data_received(HTTPPacket& packet, int length)
    {
        total_byte_count += length;

        if (state == State::Header)
        {
            op_code = static_cast<OpCode>(get_opcode());

            payload_length = get_initial_payload_length();

            if (payload_length == 126)
            {
                state = State::ExtendedPayloadLength_2;
            }
            else if (payload_length == 127)
            {
                state = State::ExtendedPayloadLength_8;
            }
            else if (is_data_masked())
            {
                state = State::MaskingKey;
            }
            else
            {
                state = State::Payload;
            }
        }
        else if (state == State::ExtendedPayloadLength_2)
        {
            payload_length = ntoh(frame_data.ext_len.len_16);
            state = State::MaskingKey;
        }
        else if (state == State::ExtendedPayloadLength_8)
        {
            payload_length = ntoh(frame_data.ext_len.len_64);
            state = State::MaskingKey;
        }
        else if (state == State::MaskingKey)
        {
            state = State::Payload;
        }
        else
        {
            auto len = static_cast<decltype(received_payload_in_current_package)>(length);
            if (is_data_masked())
            {
                for (auto i = 0u; i < len; ++i)
                {
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wconversion"
                    packet.data()[i] = packet.data()[received_payload_in_current_package + i] ^
                                       (frame_data.mask_key[(received_payload + i) % 4]);
#pragma GCC diagnostic pop
                }
            }

            received_payload += len;
            received_payload_in_current_package += len;
        }

        if (is_complete(packet))
        {
            // Resize buffer to deliver exactly the number of received bytes to the application.
            using vector_type = std::remove_reference<decltype(packet.data())>::type;
            packet.data().resize(static_cast<vector_type::size_type>(received_payload_in_current_package));

            set_message_properties(packet);
        }
    }

    OpCode WebsocketProtocol::get_opcode() const
    {
        return static_cast<OpCode>(frame_data.header[0] & 0x0F);
    }

    int WebsocketProtocol::is_data_masked() const
    {
        return frame_data.header[1] & 0x80;
    }

    uint64_t WebsocketProtocol::get_initial_payload_length() const
    {
        return frame_data.header[1] & 0x7F;
    }

    uint8_t* WebsocketProtocol::get_write_pos(HTTPPacket& packet)
    {
        if (state == State::Header)
        {
            return frame_data.header + total_byte_count;
        }
        else if (state == State::ExtendedPayloadLength_2 || state == State::ExtendedPayloadLength_8)
        {
            return reinterpret_cast<uint8_t*>(&frame_data.ext_len);
        }
        else if (state == State::MaskingKey)
        {
            return frame_data.mask_key;
        }
        else
        {
            return packet.data().data() + received_payload_in_current_package;
        }
    }

    bool WebsocketProtocol::is_complete(HTTPPacket& /*packet*/) const
    {
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-conversion"
        return state == State::Payload
               && (received_payload == payload_length
                   || received_payload_in_current_package ==
                      static_cast<decltype(received_payload_in_current_package)>(content_chunk_size));
#pragma GCC diagnostic pop
    }

    bool WebsocketProtocol::is_error()
    {
        return error;
    }

    void WebsocketProtocol::packet_consumed()
    {
        error = false;
        received_payload_in_current_package = 0;

        if (error || received_payload >= payload_length)
        {
            // All parts received
            received_payload = 0;
            total_byte_count = 0;
            state = State::Header;
        }
    }

    void WebsocketProtocol::reset()
    {
        state = State::Header;
        error = false;
        total_byte_count = 0;
        payload_length = 0;
        received_payload = 0;
        received_payload_in_current_package = 0;
    }

    void WebsocketProtocol::set_message_properties(HTTPPacket& packet)
    {
        if(received_payload <= static_cast<decltype(received_payload)>(content_chunk_size))
        {
            // This is the first part of the fragment we're forwarding to the application.
            if(received_payload < payload_length)
            {
                // There is more to come
                packet.set_continued();
            }
        }
        else
        {
            // Second or later fragment
            packet.set_continuation();

            if(received_payload < payload_length)
            {
                // Still more fragments coming.
                packet.set_continued();
            }
        }

        packet.set_ws_control_code(op_code);
    }

    bool WebsocketProtocol::is_fin_frame() const
    {
        return frame_data.header[0] & 0x80;
    }
}<commit_msg>#49 - Fix so that a package never takes more than content_chunk_size bytes of data.<commit_after>// Smooth - C++ framework for writing applications based on Espressif's ESP-IDF.
// Copyright (C) 2017 Per Malmberg (https://github.com/PerMalmberg)
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

#include <smooth/application/network/http/websocket/WebsocketProtocol.h>
#include <smooth/core/util/string_util.h>
#include <smooth/application/network/http/regular/HTTPHeaderDef.h>
#include <smooth/application/network/http/regular/RegularHTTPProtocol.h>
#include <smooth/application/network/http/regular/responses/ErrorResponse.h>
#include <smooth/core/network/util.h>

namespace smooth::application::network::http::websocket
{
    using namespace smooth::core;
    using namespace smooth::core::network;
    using namespace smooth::application::network::http::regular;

    int WebsocketProtocol::get_wanted_amount(HTTPPacket& packet)
    {
        int len;
        if (state == State::Header)
        {
            len = 2;
        }
        else if (state == State::ExtendedPayloadLength_2)
        {
            len = 2;
        }
        else if (state == State::ExtendedPayloadLength_8)
        {
            len = 8;
        }
        else if (state == State::MaskingKey)
        {
            len = 4;
        }
        else
        {
            // Websockets can use up to 64 bits (well, 63 since MSB MUST always be 0,
            // see https://tools.ietf.org/html/rfc6455#section-5.2) to specify the size.
            // Since we can't handle that large data buffers, we split it into smaller parts based on content_chunk_size

            auto remaining_to_read = payload_length - received_payload;
            auto remaining_to_fill_current =
                    static_cast<decltype(received_payload_in_current_package)>(content_chunk_size) -
                    received_payload_in_current_package;

            auto size_needed = std::min(remaining_to_read, remaining_to_fill_current);
            len = static_cast<decltype(len)>(size_needed);
            packet.ensure_room(static_cast<decltype(content_chunk_size)>(received_payload_in_current_package) + len);
        }

        return len;
    }

    void WebsocketProtocol::data_received(HTTPPacket& packet, int length)
    {
        total_byte_count += length;

        if (state == State::Header)
        {
            op_code = static_cast<OpCode>(get_opcode());

            payload_length = get_initial_payload_length();

            if (payload_length == 126)
            {
                state = State::ExtendedPayloadLength_2;
            }
            else if (payload_length == 127)
            {
                state = State::ExtendedPayloadLength_8;
            }
            else if (is_data_masked())
            {
                state = State::MaskingKey;
            }
            else
            {
                state = State::Payload;
            }
        }
        else if (state == State::ExtendedPayloadLength_2)
        {
            payload_length = ntoh(frame_data.ext_len.len_16);
            state = State::MaskingKey;
        }
        else if (state == State::ExtendedPayloadLength_8)
        {
            payload_length = ntoh(frame_data.ext_len.len_64);
            state = State::MaskingKey;
        }
        else if (state == State::MaskingKey)
        {
            state = State::Payload;
        }
        else
        {
            auto len = static_cast<decltype(received_payload_in_current_package)>(length);
            if (is_data_masked())
            {
                for (decltype(len) i = 0; i < len; ++i)
                {
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wconversion"
                    packet.data()[i] = packet.data()[received_payload_in_current_package + i] ^
                                       (frame_data.mask_key[(received_payload + i) % 4]);
#pragma GCC diagnostic pop
                }
            }

            received_payload += len;
            received_payload_in_current_package += len;
        }

        if (is_complete(packet))
        {
            // Resize buffer to deliver exactly the number of received bytes to the application.
            using vector_type = std::remove_reference<decltype(packet.data())>::type;
            packet.data().resize(static_cast<vector_type::size_type>(received_payload_in_current_package));

            set_message_properties(packet);
        }
    }

    OpCode WebsocketProtocol::get_opcode() const
    {
        return static_cast<OpCode>(frame_data.header[0] & 0x0F);
    }

    int WebsocketProtocol::is_data_masked() const
    {
        return frame_data.header[1] & 0x80;
    }

    uint64_t WebsocketProtocol::get_initial_payload_length() const
    {
        return frame_data.header[1] & 0x7F;
    }

    uint8_t* WebsocketProtocol::get_write_pos(HTTPPacket& packet)
    {
        if (state == State::Header)
        {
            return frame_data.header + total_byte_count;
        }
        else if (state == State::ExtendedPayloadLength_2 || state == State::ExtendedPayloadLength_8)
        {
            return reinterpret_cast<uint8_t*>(&frame_data.ext_len);
        }
        else if (state == State::MaskingKey)
        {
            return frame_data.mask_key;
        }
        else
        {
            return packet.data().data() + received_payload_in_current_package;
        }
    }

    bool WebsocketProtocol::is_complete(HTTPPacket& /*packet*/) const
    {
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-conversion"
        return state == State::Payload
               && (received_payload == payload_length
                   || received_payload_in_current_package ==
                      static_cast<decltype(received_payload_in_current_package)>(content_chunk_size));
#pragma GCC diagnostic pop
    }

    bool WebsocketProtocol::is_error()
    {
        return error;
    }

    void WebsocketProtocol::packet_consumed()
    {
        error = false;
        received_payload_in_current_package = 0;

        if (error || received_payload >= payload_length)
        {
            // All parts received
            received_payload = 0;
            total_byte_count = 0;
            state = State::Header;
        }
    }

    void WebsocketProtocol::reset()
    {
        state = State::Header;
        error = false;
        total_byte_count = 0;
        payload_length = 0;
        received_payload = 0;
        received_payload_in_current_package = 0;
    }

    void WebsocketProtocol::set_message_properties(HTTPPacket& packet)
    {
        if (received_payload <= static_cast<decltype(received_payload)>(content_chunk_size))
        {
            // This is the first part of the fragment we're forwarding to the application.
            if (received_payload < payload_length)
            {
                // There is more to come
                packet.set_continued();
            }
        }
        else
        {
            // Second or later fragment
            packet.set_continuation();

            if (received_payload < payload_length)
            {
                // Still more fragments coming.
                packet.set_continued();
            }
        }

        packet.set_ws_control_code(op_code);
    }

    bool WebsocketProtocol::is_fin_frame() const
    {
        return frame_data.header[0] & 0x80;
    }
}<|endoftext|>


<commit_before>/*
 * (C) Copyright 2014 Kurento (http://kurento.org/)
 *
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the GNU Lesser General Public License
 * (LGPL) version 2.1 which accompanies this distribution, and is available at
 * http://www.gnu.org/licenses/lgpl-2.1.html
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 */

#include <gst/gst.h>
#include "RabbitMQPipeline.hpp"
#include "RabbitMQEventHandler.hpp"

#include <MediaObjectImpl.hpp>
#include <MediaSet.hpp>

#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>

#define GST_CAT_DEFAULT kurento_rabbitmq_pipeline
GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
#define GST_DEFAULT_NAME "KurentoRabbitMQPipeline"

/* This is included to avoid problems with slots and lamdas */
#include <type_traits>
#include <sigc++/sigc++.h>
#include <event2/event_struct.h>
namespace sigc
{
template <typename Functor>
struct functor_trait<Functor, false> {
  typedef decltype (::sigc::mem_fun (std::declval<Functor &> (),
                                     &Functor::operator() ) ) _intermediate;

  typedef typename _intermediate::result_type result_type;
  typedef Functor functor_type;
};
}

// #define PIPELINE_QUEUE_PREFIX "media_pipeline_"
#define PIPELINE_QUEUE_PREFIX ""
#define EVENT_EXCHANGE_PREFIX "event_"

#define PIPELINE_QUEUE_TTL 240000

namespace kurento
{

static std::string
generateUUID()
{
  std::stringstream ss;
  boost::uuids::uuid uuid = boost::uuids::random_generator() ();

  ss << uuid;
  return ss.str();
}

RabbitMQPipeline::RabbitMQPipeline (Glib::KeyFile &confFile,
                                    const std::string &address, const int port)
{
  httpService = std::shared_ptr<HttpService> (new HttpService (
                  confFile, /* fixedPort */ false) );
  httpService->start();
  setConfig (address, port);

  MediaSet::getMediaSet().signalEmpty.connect ([] () {
    GST_INFO ("Mediaset is empty, exiting from child process");
    kill (getpid(), SIGINT);
  });
}

RabbitMQPipeline::~RabbitMQPipeline()
{
  stopListen();
  httpService->stop();
}

void
RabbitMQPipeline::processMessage (RabbitMQMessage &message)
{
  std::string data = message.getData();
  std::string response;

  GST_DEBUG ("Message: >%s<", data.c_str() );
  process (data, response);
  GST_DEBUG ("Response: >%s<", response.c_str() );

  message.ack();
  message.reply (response);
}

void
RabbitMQPipeline::startRequest (RabbitMQMessage &message)
{
  Json::Value responseJson;
  Json::Reader reader;
  std::string response;
  std::string request = message.getData();

  GST_DEBUG ("Message: >%s<", request.c_str() );
  process (request, response);

  reader.parse (response, responseJson);
  message.ack();

  if (responseJson.isObject() && responseJson.isMember ("result")
      && responseJson["result"].isObject()
      && responseJson["result"].isMember ("value") ) {
    pipelineId = responseJson["result"]["value"].asString();

    listenQueue (PIPELINE_QUEUE_PREFIX + pipelineId, false, PIPELINE_QUEUE_TTL);
    getConnection()->declareExchange (EVENT_EXCHANGE_PREFIX + pipelineId,
                                      RabbitMQConnection::EXCHANGE_TYPE_FANOUT,
                                      false, PIPELINE_QUEUE_TTL);
  }

  message.reply (getConnection(), response);

  GST_DEBUG ("Response: >%s<", response.c_str() );
}

std::string
RabbitMQPipeline::connectEventHandler (std::shared_ptr< MediaObject > obj,
                                       const std::string &sessionId,
                                       const std::string &eventType,
                                       const Json::Value &params)
{
  std::string subscriptionId;
  std::shared_ptr <MediaObjectImpl> object = std::dynamic_pointer_cast
      <MediaObjectImpl> (obj);
  std::string eventId = object->getId() + "/" + eventType;
  std::shared_ptr <EventHandler> handler;
  Glib::Threads::Mutex::Lock lock (mutex);

  if (handlers.find (eventId) != handlers.end() ) {
    handler = handlers[eventId].lock();
  }

  if (!handler) {
    handler = std::shared_ptr <EventHandler> (new RabbitMQEventHandler (obj,
              getConnection()->getAddress(), getConnection()->getPort(),
              EVENT_EXCHANGE_PREFIX + pipelineId, eventId),
              std::bind (&RabbitMQPipeline::destroyHandler, this, std::placeholders::_1) );

    subscriptionId = ServerMethods::connectEventHandler (obj, sessionId, eventType,
                     handler);
    handlers[eventId] = std::weak_ptr <EventHandler> (handler);
  } else {
    subscriptionId = generateUUID();
    registerEventHandler (obj, sessionId, subscriptionId, handler);
  }

  return subscriptionId;
}

void RabbitMQPipeline::destroyHandler (EventHandler *handler)
{
  RabbitMQEventHandler *rabbitHandler = dynamic_cast <RabbitMQEventHandler *>
                                        (handler);

  if (rabbitHandler != NULL) {
    std::string eventId;
    Glib::Threads::Mutex::Lock lock (mutex);

    eventId = rabbitHandler->getRoutingKey();

    if (handlers.find (eventId) != handlers.end() ) {
      handlers.erase (eventId);
    }
  }

  delete handler;
}

RabbitMQPipeline::StaticConstructor RabbitMQPipeline::staticConstructor;

RabbitMQPipeline::StaticConstructor::StaticConstructor()
{
  GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, GST_DEFAULT_NAME, 0,
                           GST_DEFAULT_NAME);
}

} /* kurento */
<commit_msg>RabbitMQPipeline: Kill child process if pipeline cannot be created<commit_after>/*
 * (C) Copyright 2014 Kurento (http://kurento.org/)
 *
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the GNU Lesser General Public License
 * (LGPL) version 2.1 which accompanies this distribution, and is available at
 * http://www.gnu.org/licenses/lgpl-2.1.html
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 */

#include <gst/gst.h>
#include "RabbitMQPipeline.hpp"
#include "RabbitMQEventHandler.hpp"

#include <MediaObjectImpl.hpp>
#include <MediaSet.hpp>

#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>

#define GST_CAT_DEFAULT kurento_rabbitmq_pipeline
GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
#define GST_DEFAULT_NAME "KurentoRabbitMQPipeline"

/* This is included to avoid problems with slots and lamdas */
#include <type_traits>
#include <sigc++/sigc++.h>
#include <event2/event_struct.h>
namespace sigc
{
template <typename Functor>
struct functor_trait<Functor, false> {
  typedef decltype (::sigc::mem_fun (std::declval<Functor &> (),
                                     &Functor::operator() ) ) _intermediate;

  typedef typename _intermediate::result_type result_type;
  typedef Functor functor_type;
};
}

// #define PIPELINE_QUEUE_PREFIX "media_pipeline_"
#define PIPELINE_QUEUE_PREFIX ""
#define EVENT_EXCHANGE_PREFIX "event_"

#define PIPELINE_QUEUE_TTL 240000

namespace kurento
{

static std::string
generateUUID()
{
  std::stringstream ss;
  boost::uuids::uuid uuid = boost::uuids::random_generator() ();

  ss << uuid;
  return ss.str();
}

RabbitMQPipeline::RabbitMQPipeline (Glib::KeyFile &confFile,
                                    const std::string &address, const int port)
{
  httpService = std::shared_ptr<HttpService> (new HttpService (
                  confFile, /* fixedPort */ false) );
  httpService->start();
  setConfig (address, port);

  MediaSet::getMediaSet().signalEmpty.connect ([] () {
    GST_INFO ("Mediaset is empty, exiting from child process");
    kill (getpid(), SIGINT);
  });
}

RabbitMQPipeline::~RabbitMQPipeline()
{
  stopListen();
  httpService->stop();
}

void
RabbitMQPipeline::processMessage (RabbitMQMessage &message)
{
  std::string data = message.getData();
  std::string response;

  GST_DEBUG ("Message: >%s<", data.c_str() );
  process (data, response);
  GST_DEBUG ("Response: >%s<", response.c_str() );

  message.ack();
  message.reply (response);
}

void
RabbitMQPipeline::startRequest (RabbitMQMessage &message)
{
  Json::Value responseJson;
  Json::Reader reader;
  std::string response;
  std::string request = message.getData();

  GST_DEBUG ("Message: >%s<", request.c_str() );
  process (request, response);

  reader.parse (response, responseJson);
  message.ack();

  if (responseJson.isObject() && responseJson.isMember ("result")
      && responseJson["result"].isObject()
      && responseJson["result"].isMember ("value") ) {
    pipelineId = responseJson["result"]["value"].asString();

    listenQueue (PIPELINE_QUEUE_PREFIX + pipelineId, false, PIPELINE_QUEUE_TTL);
    getConnection()->declareExchange (EVENT_EXCHANGE_PREFIX + pipelineId,
                                      RabbitMQConnection::EXCHANGE_TYPE_FANOUT,
                                      false, PIPELINE_QUEUE_TTL);
  }

  message.reply (getConnection(), response);

  GST_DEBUG ("Response: >%s<", response.c_str() );

  if (MediaSet::getMediaSet().empty() ) {
    GST_ERROR ("Error creating media pipeline, terminating process");
    kill (getpid(), SIGINT);
  }
}

std::string
RabbitMQPipeline::connectEventHandler (std::shared_ptr< MediaObject > obj,
                                       const std::string &sessionId,
                                       const std::string &eventType,
                                       const Json::Value &params)
{
  std::string subscriptionId;
  std::shared_ptr <MediaObjectImpl> object = std::dynamic_pointer_cast
      <MediaObjectImpl> (obj);
  std::string eventId = object->getId() + "/" + eventType;
  std::shared_ptr <EventHandler> handler;
  Glib::Threads::Mutex::Lock lock (mutex);

  if (handlers.find (eventId) != handlers.end() ) {
    handler = handlers[eventId].lock();
  }

  if (!handler) {
    handler = std::shared_ptr <EventHandler> (new RabbitMQEventHandler (obj,
              getConnection()->getAddress(), getConnection()->getPort(),
              EVENT_EXCHANGE_PREFIX + pipelineId, eventId),
              std::bind (&RabbitMQPipeline::destroyHandler, this, std::placeholders::_1) );

    subscriptionId = ServerMethods::connectEventHandler (obj, sessionId, eventType,
                     handler);
    handlers[eventId] = std::weak_ptr <EventHandler> (handler);
  } else {
    subscriptionId = generateUUID();
    registerEventHandler (obj, sessionId, subscriptionId, handler);
  }

  return subscriptionId;
}

void RabbitMQPipeline::destroyHandler (EventHandler *handler)
{
  RabbitMQEventHandler *rabbitHandler = dynamic_cast <RabbitMQEventHandler *>
                                        (handler);

  if (rabbitHandler != NULL) {
    std::string eventId;
    Glib::Threads::Mutex::Lock lock (mutex);

    eventId = rabbitHandler->getRoutingKey();

    if (handlers.find (eventId) != handlers.end() ) {
      handlers.erase (eventId);
    }
  }

  delete handler;
}

RabbitMQPipeline::StaticConstructor RabbitMQPipeline::staticConstructor;

RabbitMQPipeline::StaticConstructor::StaticConstructor()
{
  GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, GST_DEFAULT_NAME, 0,
                           GST_DEFAULT_NAME);
}

} /* kurento */
<|endoftext|>


<commit_before>#include <atomic>
#include <memory>
#include <string>
#include <thread>

#include <boost/program_options/cmdline.hpp>
#include <boost/program_options/options_description.hpp>
#include <boost/program_options/parsers.hpp>
#include <boost/program_options/variables_map.hpp>

#include "jml/arch/exception.h"
#include "soa/types/date.h"
#include "soa/types/value_description.h"
#include "soa/utils/print_utils.h"
#include "soa/service/http_client.h"
#include "soa/service/http_endpoint.h"
#include "soa/service/named_endpoint.h"
#include "soa/service/message_loop.h"
#include "soa/service/rest_proxy.h"
#include "soa/service/rest_service_endpoint.h"
#include "soa/service/runner.h"

#include "test_http_services.h"

using namespace std;
using namespace Datacratic;

enum HttpMethod {
    GET,
    POST,
    PUT
};

/* bench methods */

double
AsyncModelBench(HttpMethod method,
                const string & baseUrl, const string & payload,
                int maxReqs, int concurrency)
{
    int numReqs, numResponses(0), numMissed(0);
    MessageLoop loop(1, 0, -1);

    loop.start();

    auto client = make_shared<HttpClient>(baseUrl, concurrency);
    loop.addSource("httpClient", client);

    auto onResponse = [&] (const HttpRequest & rq, HttpClientError errorCode_,
                           int status, string && headers, string && body) {
        numResponses++;
        // if (numResponses % 1000) {
            // cerr << "resps: "  + to_string(numResponses) + "\n";
        // }
        if (numResponses == maxReqs) {
            // cerr << "received all responses\n";
            ML::futex_wake(numResponses);
        }
    };
    auto cbs = make_shared<HttpClientSimpleCallbacks>(onResponse);
    HttpRequest::Content content(payload, "application/binary");

    auto & clientRef = *client.get();
    string url("/");
    Date start = Date::now();
    for (numReqs = 0; numReqs < maxReqs;) {
        bool result;
        if (method == GET) {
            result = clientRef.get(url, cbs);
        }
        else if (method == POST) {
            result = clientRef.post(url, cbs, content);
        }
        else if (method == PUT) {
            result = clientRef.put(url, cbs, content);
        }
        else {
            result = true;
        }
        if (result) {
            numReqs++;
            // if (numReqs % 1000) {
            //     cerr << "reqs: "  + to_string(numReqs) + "\n";
            // }
        }
        else {
            numMissed++;
        }
    }

    while (numResponses < maxReqs) {
        // cerr << (" num Responses: " + to_string(numResponses)
        //          + "; max reqs: " + to_string(maxReqs)
        //          + "\n");
        int old(numResponses);
        ML::futex_wait(numResponses, old);
    }
    Date end = Date::now();

    loop.removeSource(client.get());
    client->waitConnectionState(AsyncEventSource::DISCONNECTED);

    cerr << "num misses: "  + to_string(numMissed) + "\n";

    return end - start;
}

double
ThreadedModelBench(HttpMethod method,
                   const string & baseUrl, const string & payload,
                   int maxReqs, int concurrency)
{
    vector<thread> threads;

    HttpRestProxy::Content content(payload, "application/binary");

    auto threadFn = [&] (int num, int nReqs) {
        int i;
        HttpRestProxy client(baseUrl);
        for (i = 0; i < nReqs; i++) {
            if (method == GET) {
                auto response = client.get("/");
            }
            else if (method == POST) {
                auto response = client.post("/", content);
            }
            else if (method == PUT) {
                auto response = client.put("/", content);
            }
        }
    };

    Date start = Date::now();
    int slice(maxReqs / concurrency);
    for (int i = 0; i < concurrency; i++) {
        // cerr << "doing slice: "  + to_string(slice) + "\n";
        threads.emplace_back(threadFn, i, slice);
    }
    for (int i = 0; i < concurrency; i++) {
        threads[i].join();
    }

    return Date::now() - start;
}

int main(int argc, char *argv[])
{
    using namespace boost::program_options;

    size_t concurrency(0);
    int model(0);
    size_t maxReqs(0);
    string method("GET");
    size_t payloadSize(0);

    string serveriface("127.0.0.1");
    string clientiface(serveriface);

    options_description all_opt;
    all_opt.add_options()
        ("client-iface,C", value(&clientiface),
         "address:port to connect to (\"none\" for no client)")
        ("concurrency,c", value(&concurrency),
         "Number of concurrent requests")
        ("method,M", value(&method),
         "Method to use (\"GET\"*, \"PUT\", \"POST\")")
        ("model,m", value(&model),
         "Type of concurrency model (1 for async, 2 for threaded))")
        ("requests,r", value(&maxReqs),
         "total of number of requests to perform")
        ("payload-size,s", value(&payloadSize),
         "size of the response body")
        ("server-iface,S", value(&serveriface),
         "server address (\"none\" for no server)")
        ("help,H", "show help");

    if (argc == 1) {
        return 0;
    }

    variables_map vm;
    store(command_line_parser(argc, argv)
          .options(all_opt)
          .run(),
          vm);
    notify(vm);

    if (vm.count("help")) {
        cerr << all_opt << endl;
        return 1;
    }

    /* service setup */
    auto proxies = make_shared<ServiceProxies>();

    HttpGetService service(proxies);

    if (concurrency == 0) {
        throw ML::Exception("'concurrency' must be specified");
    }

    if (payloadSize == 0) {
        throw ML::Exception("'payload-size' must be specified");
    }

    string payload;
    while (payload.size() < payloadSize) {
        payload += randomString(128);
    }

    if (serveriface != "none") {
        cerr << "launching server\n";
        service.portToUse = 20000;

        service.addResponse("GET", "/", 200, payload);
        service.addResponse("PUT", "/", 200, "");
        service.addResponse("POST", "/", 200, "");
        service.start(serveriface, concurrency);
    }

    if (clientiface != "none") {
        cerr << "launching client\n";
        if (maxReqs == 0) {
            throw ML::Exception("'max-reqs' must be specified");
        }

        if (!(method == "GET" || method == "POST" || method == "PUT")) {
            throw ML::Exception("invalid method:" + method);
        }

        string baseUrl;
        if (serveriface != "none") {
            baseUrl = ("http://" + serveriface
                       + ":" + to_string(service.port()));
        }
        else {
            baseUrl = "http://" + clientiface;
        }

        ::printf("model\tconc.\treqs\tsize\ttime_secs\tqps\n");

        HttpMethod httpMethod;
        if (method == "GET") {
            httpMethod = GET;
        }
        else if (method == "POST") {
            httpMethod = POST;
        }
        else if (method == "PUT") {
            httpMethod = PUT;
        }
        else {
            throw ML::Exception("unknown method: "  + method);
        }

        double delta;
        if (model == 1) {
            delta = AsyncModelBench(httpMethod, baseUrl, payload, maxReqs, concurrency);
        }
        else if (model == 2) {
            delta = ThreadedModelBench(httpMethod, baseUrl, payload, maxReqs, concurrency);
        }
        else {
            throw ML::Exception("invalid 'model'");
        }
        double qps = maxReqs / delta;
        ::printf("%d\t%lu\t%lu\t%lu\t%f\t%f\n",
                 model, concurrency, maxReqs, payloadSize, delta, qps);
    }
    else {
        while (1) {
            sleep(100);
        }
    }

    struct rusage usage;
    getrusage(RUSAGE_SELF, &usage);
    Json::Value jsonUsage = jsonEncode(usage);
    cerr << "rusage:\n" << jsonUsage.toStyledString() << endl;

    return 0;
}
<commit_msg>http_client_bench: report bytes per second<commit_after>#include <atomic>
#include <memory>
#include <string>
#include <thread>

#include <boost/program_options/cmdline.hpp>
#include <boost/program_options/options_description.hpp>
#include <boost/program_options/parsers.hpp>
#include <boost/program_options/variables_map.hpp>

#include "jml/arch/exception.h"
#include "soa/types/date.h"
#include "soa/types/value_description.h"
#include "soa/utils/print_utils.h"
#include "soa/service/http_client.h"
#include "soa/service/http_endpoint.h"
#include "soa/service/named_endpoint.h"
#include "soa/service/message_loop.h"
#include "soa/service/rest_proxy.h"
#include "soa/service/rest_service_endpoint.h"
#include "soa/service/runner.h"

#include "test_http_services.h"

using namespace std;
using namespace Datacratic;

enum HttpMethod {
    GET,
    POST,
    PUT
};

/* bench methods */

double
AsyncModelBench(HttpMethod method,
                const string & baseUrl, const string & payload,
                int maxReqs, int concurrency)
{
    int numReqs, numResponses(0), numMissed(0);
    MessageLoop loop(1, 0, -1);

    loop.start();

    auto client = make_shared<HttpClient>(baseUrl, concurrency);
    loop.addSource("httpClient", client);

    auto onResponse = [&] (const HttpRequest & rq, HttpClientError errorCode_,
                           int status, string && headers, string && body) {
        numResponses++;
        // if (numResponses % 1000) {
            // cerr << "resps: "  + to_string(numResponses) + "\n";
        // }
        if (numResponses == maxReqs) {
            // cerr << "received all responses\n";
            ML::futex_wake(numResponses);
        }
    };
    auto cbs = make_shared<HttpClientSimpleCallbacks>(onResponse);
    HttpRequest::Content content(payload, "application/binary");

    auto & clientRef = *client.get();
    string url("/");
    Date start = Date::now();
    for (numReqs = 0; numReqs < maxReqs;) {
        bool result;
        if (method == GET) {
            result = clientRef.get(url, cbs);
        }
        else if (method == POST) {
            result = clientRef.post(url, cbs, content);
        }
        else if (method == PUT) {
            result = clientRef.put(url, cbs, content);
        }
        else {
            result = true;
        }
        if (result) {
            numReqs++;
            // if (numReqs % 1000) {
            //     cerr << "reqs: "  + to_string(numReqs) + "\n";
            // }
        }
        else {
            numMissed++;
        }
    }

    while (numResponses < maxReqs) {
        // cerr << (" num Responses: " + to_string(numResponses)
        //          + "; max reqs: " + to_string(maxReqs)
        //          + "\n");
        int old(numResponses);
        ML::futex_wait(numResponses, old);
    }
    Date end = Date::now();

    loop.removeSource(client.get());
    client->waitConnectionState(AsyncEventSource::DISCONNECTED);

    cerr << "num misses: "  + to_string(numMissed) + "\n";

    return end - start;
}

double
ThreadedModelBench(HttpMethod method,
                   const string & baseUrl, const string & payload,
                   int maxReqs, int concurrency)
{
    vector<thread> threads;

    HttpRestProxy::Content content(payload, "application/binary");

    auto threadFn = [&] (int num, int nReqs) {
        int i;
        HttpRestProxy client(baseUrl);
        for (i = 0; i < nReqs; i++) {
            if (method == GET) {
                auto response = client.get("/");
            }
            else if (method == POST) {
                auto response = client.post("/", content);
            }
            else if (method == PUT) {
                auto response = client.put("/", content);
            }
        }
    };

    Date start = Date::now();
    int slice(maxReqs / concurrency);
    for (int i = 0; i < concurrency; i++) {
        // cerr << "doing slice: "  + to_string(slice) + "\n";
        threads.emplace_back(threadFn, i, slice);
    }
    for (int i = 0; i < concurrency; i++) {
        threads[i].join();
    }

    return Date::now() - start;
}

int main(int argc, char *argv[])
{
    using namespace boost::program_options;

    size_t concurrency(0);
    int model(0);
    size_t maxReqs(0);
    string method("GET");
    size_t payloadSize(0);

    string serveriface("127.0.0.1");
    string clientiface(serveriface);

    options_description all_opt;
    all_opt.add_options()
        ("client-iface,C", value(&clientiface),
         "address:port to connect to (\"none\" for no client)")
        ("concurrency,c", value(&concurrency),
         "Number of concurrent requests")
        ("method,M", value(&method),
         "Method to use (\"GET\"*, \"PUT\", \"POST\")")
        ("model,m", value(&model),
         "Type of concurrency model (1 for async, 2 for threaded))")
        ("requests,r", value(&maxReqs),
         "total of number of requests to perform")
        ("payload-size,s", value(&payloadSize),
         "size of the response body")
        ("server-iface,S", value(&serveriface),
         "server address (\"none\" for no server)")
        ("help,H", "show help");

    if (argc == 1) {
        return 0;
    }

    variables_map vm;
    store(command_line_parser(argc, argv)
          .options(all_opt)
          .run(),
          vm);
    notify(vm);

    if (vm.count("help")) {
        cerr << all_opt << endl;
        return 1;
    }

    /* service setup */
    auto proxies = make_shared<ServiceProxies>();

    HttpGetService service(proxies);

    if (concurrency == 0) {
        throw ML::Exception("'concurrency' must be specified");
    }

    if (payloadSize == 0) {
        throw ML::Exception("'payload-size' must be specified");
    }

    string payload;
    while (payload.size() < payloadSize) {
        payload += randomString(128);
    }

    if (serveriface != "none") {
        cerr << "launching server\n";
        service.portToUse = 20000;

        service.addResponse("GET", "/", 200, payload);
        service.addResponse("PUT", "/", 200, "");
        service.addResponse("POST", "/", 200, "");
        service.start(serveriface, concurrency);
    }

    if (clientiface != "none") {
        cerr << "launching client\n";
        if (maxReqs == 0) {
            throw ML::Exception("'max-reqs' must be specified");
        }

        if (!(method == "GET" || method == "POST" || method == "PUT")) {
            throw ML::Exception("invalid method:" + method);
        }

        string baseUrl;
        if (serveriface != "none") {
            baseUrl = ("http://" + serveriface
                       + ":" + to_string(service.port()));
        }
        else {
            baseUrl = "http://" + clientiface;
        }

        ::printf("model\tconc.\treqs\tsize\ttime_secs\tBps\tqps\n");

        HttpMethod httpMethod;
        if (method == "GET") {
            httpMethod = GET;
        }
        else if (method == "POST") {
            httpMethod = POST;
        }
        else if (method == "PUT") {
            httpMethod = PUT;
        }
        else {
            throw ML::Exception("unknown method: "  + method);
        }

        double delta;
        if (model == 1) {
            delta = AsyncModelBench(httpMethod, baseUrl, payload, maxReqs, concurrency);
        }
        else if (model == 2) {
            delta = ThreadedModelBench(httpMethod, baseUrl, payload, maxReqs, concurrency);
        }
        else {
            throw ML::Exception("invalid 'model'");
        }
        double qps = maxReqs / delta;
        double bps = double(maxReqs * payload.size()) / delta;
        ::printf("%d\t%lu\t%lu\t%lu\t%f\t%f\t%f\n",
                 model, concurrency, maxReqs, payloadSize, delta, bps, qps);
    }
    else {
        while (1) {
            sleep(100);
        }
    }

    struct rusage usage;
    getrusage(RUSAGE_SELF, &usage);
    Json::Value jsonUsage = jsonEncode(usage);
    cerr << "rusage:\n" << jsonUsage.toStyledString() << endl;

    return 0;
}
<|endoftext|>


<commit_before>/*************** <auto-copyright.pl BEGIN do not edit this line> **************
 *
 * VR Juggler is (C) Copyright 1998-2005 by Iowa State University
 *
 * Original Authors:
 *   Allen Bierbaum, Christopher Just,
 *   Patrick Hartling, Kevin Meinert,
 *   Carolina Cruz-Neira, Albert Baker
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 02111-1307, USA.
 *
 * -----------------------------------------------------------------
 * File:          $RCSfile$
 * Date modified: $Date$
 * Version:       $Revision$
 * -----------------------------------------------------------------
 *
 *************** <auto-copyright.pl END do not edit this line> ***************/

#include <gadget/Devices/DriverConfig.h>

#include <vector>
#include <string>

#include <vpr/vpr.h>
#include <vpr/Thread/Thread.h>
#include <vpr/System.h>

#include <gadget/InputManager.h>
#include <gadget/Type/Input.h>
#include <gadget/Type/Digital.h>
#include <gadget/Type/InputMixer.h>
#include <gadget/Type/DeviceConstructor.h>

using namespace gadget;

/** The device driver class. */
class MyButtonDevice : public InputMixer<Input, Digital>
{
public:
   MyButtonDevice()
      : mSampleThread(NULL)
      , mRunning(false)
   {
      /* Do nothing. */ ;
   }

   virtual ~MyButtonDevice()
   {
      if ( mRunning )
      {
         this->stopSampling();
      }
   }

   virtual bool config(jccl::ConfigElementPtr e);
   virtual void updateData();

   virtual bool startSampling();
   virtual bool sample();
   virtual bool stopSampling();

   static std::string getElementType();

   /**
    * Invokes the global scope delete operator.  This is required for proper
    * releasing of memory in DLLs on Win32.
    */
   void operator delete(void* p)
   {
      ::operator delete(p);
   }

protected:
   /**
    * Deletes this object.  This is an implementation of the pure virtual
    * gadget::Input::destroy() method.
    */
   virtual void destroy()
   {
      delete this;
   }

private:
   static void   threadedSampleFunction( void* classPointer );
   int           mDigitalData;
   vpr::Thread*  mSampleThread;
   bool          mRunning;

   // configuration data set by config()
   std::string   mPort;
   int           mBaud;
};

/** Entry point function for the device driver plug-in. */
extern "C" GADGET_DRIVER_EXPORT(void) initDevice(InputManager* inputMgr)
{
   new DeviceConstructor<MyButtonDevice>(inputMgr);
}

/** Returns a string that matches this device's configuration element type. */
std::string MyButtonDevice::getElementType()
{
   return std::string("MyButtonDevice");
}

//: When the system detects a configuration change for your driver, it will
//  pass the new jccl::ConfigElement into this function.  See the documentation
//  on config elements, for information on how to access them.
bool MyButtonDevice::config(jccl::ConfigElementPtr e)
{
  if (!Digital::config(e))
  {
     return false;
  }

  mPort = e->getProperty<std::string>("port");
  mBaud = e->getProperty<int>("baud");

  return true;
}

void MyButtonDevice::updateData()
{
   if ( mRunning )
   {
      swapDigitalBuffers();
   }
}

/**
 * Spanws the sample thread, which calls MyButtonDevice::sample() repeatedly.
 */
bool MyButtonDevice::startSampling()
{
   mRunning = true;
   mSampleThread = new vpr::Thread(threadedSampleFunction, (void*) this);

   if ( ! mSampleThread->valid() )
   {
      mRunning = false;
      return false; // thread creation failed
   }
   else
   {
      return true; // thread creation success
   }
}

/**
 * Records (or samples) the current data.  This is called repeatedly by the
 * sample thread created by startSampling().
 */
bool MyButtonDevice::sample()
{
   bool status(false);

   if ( mRunning )
   {
      // Here you would add your code to sample the hardware for a button
      // press:
      std::vector<DigitalData> samples(1);
      samples[0] = 1;
      addDigitalSample(samples);

      // Successful sample.
      status = true;
   }

   return status;
}

/** Kills the sample thread. */
bool MyButtonDevice::stopSampling()
{
   mRunning = false;

   if (mSampleThread != NULL)
   {
      mSampleThread->kill(); // Not guaranteed to work on all platforms
      mSampleThread->join();
      delete mSampleThread;
      mSampleThread = NULL;
   }
   return true;
}

/**
 * Our sampling function that is executed by the spawned sample thread.
 * This function is declared as a static member of MyButtonDevice.  It simply
 * calls MyButtonDevice::sample() over and over.
 */
void MyButtonDevice::threadedSampleFunction(void* classPointer)
{
   MyButtonDevice* this_ptr = static_cast<MyButtonDevice*>( classPointer );

   // spin until someone kills "mSampleThread"
   while ( this_ptr->mRunning )
   {
     this_ptr->sample();
     vpr::System::sleep(1); //specify some time here, so you don't waste CPU cycles
   }
}
<commit_msg>More fixes for this "driver."  Since it is not a real driver, it is always lagging behind all the others.<commit_after>/*************** <auto-copyright.pl BEGIN do not edit this line> **************
 *
 * VR Juggler is (C) Copyright 1998-2005 by Iowa State University
 *
 * Original Authors:
 *   Allen Bierbaum, Christopher Just,
 *   Patrick Hartling, Kevin Meinert,
 *   Carolina Cruz-Neira, Albert Baker
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 02111-1307, USA.
 *
 * -----------------------------------------------------------------
 * File:          $RCSfile$
 * Date modified: $Date$
 * Version:       $Revision$
 * -----------------------------------------------------------------
 *
 *************** <auto-copyright.pl END do not edit this line> ***************/

#include <gadget/Devices/DriverConfig.h>

#include <vector>
#include <string>

#include <vpr/vpr.h>
#include <vpr/Thread/Thread.h>
#include <vpr/System.h>

#include <gadget/gadgetParam.h>
#include <gadget/InputManager.h>
#include <gadget/Type/Input.h>
#include <gadget/Type/Digital.h>
#include <gadget/Type/InputMixer.h>
#include <gadget/Type/DeviceConstructor.h>

using namespace gadget;

/** The device driver class. */
class MyButtonDevice : public InputMixer<Input, Digital>
{
public:
   MyButtonDevice()
      : mSampleThread(NULL)
      , mRunning(false)
   {
      /* Do nothing. */ ;
   }

   virtual ~MyButtonDevice()
   {
      if ( mRunning )
      {
         this->stopSampling();
      }
   }

   virtual bool config(jccl::ConfigElementPtr e);
   virtual void updateData();

   virtual bool startSampling();
   virtual bool sample();
   virtual bool stopSampling();

   static std::string getElementType();

   /**
    * Invokes the global scope delete operator.  This is required for proper
    * releasing of memory in DLLs on Win32.
    */
   void operator delete(void* p)
   {
      ::operator delete(p);
   }

protected:
   /**
    * Deletes this object.  This is an implementation of the pure virtual
    * gadget::Input::destroy() method.
    */
   virtual void destroy()
   {
      delete this;
   }

private:
   static void   threadedSampleFunction( void* classPointer );
   int           mDigitalData;
   vpr::Thread*  mSampleThread;
   bool          mRunning;

   // configuration data set by config()
   std::string   mPort;
   int           mBaud;
};

/** Entry point function for the device driver plug-in. */
extern "C" GADGET_DRIVER_EXPORT(vpr::Uint32) getGadgeteerVersion()
{
   return __GADGET_version;
}

/** Entry point function for the device driver plug-in. */
extern "C" GADGET_DRIVER_EXPORT(void) initDevice(InputManager* inputMgr)
{
   new DeviceConstructor<MyButtonDevice>(inputMgr);
}

/** Returns a string that matches this device's configuration element type. */
std::string MyButtonDevice::getElementType()
{
   return std::string("button_device");
}

//: When the system detects a configuration change for your driver, it will
//  pass the new jccl::ConfigElement into this function.  See the documentation
//  on config elements, for information on how to access them.
bool MyButtonDevice::config(jccl::ConfigElementPtr e)
{
  if (!Digital::config(e))
  {
     return false;
  }

  mPort = e->getProperty<std::string>("port");
  mBaud = e->getProperty<int>("baud");

  return true;
}

void MyButtonDevice::updateData()
{
   if ( mRunning )
   {
      swapDigitalBuffers();
   }
}

/**
 * Spanws the sample thread, which calls MyButtonDevice::sample() repeatedly.
 */
bool MyButtonDevice::startSampling()
{
   mRunning = true;
   mSampleThread = new vpr::Thread(threadedSampleFunction, (void*) this);

   if ( ! mSampleThread->valid() )
   {
      mRunning = false;
      return false; // thread creation failed
   }
   else
   {
      return true; // thread creation success
   }
}

/**
 * Records (or samples) the current data.  This is called repeatedly by the
 * sample thread created by startSampling().
 */
bool MyButtonDevice::sample()
{
   bool status(false);

   if ( mRunning )
   {
      // Here you would add your code to sample the hardware for a button
      // press:
      std::vector<DigitalData> samples(1);
      samples[0] = 1;
      addDigitalSample(samples);

      // Successful sample.
      status = true;
   }

   return status;
}

/** Kills the sample thread. */
bool MyButtonDevice::stopSampling()
{
   mRunning = false;

   if (mSampleThread != NULL)
   {
      mSampleThread->kill(); // Not guaranteed to work on all platforms
      mSampleThread->join();
      delete mSampleThread;
      mSampleThread = NULL;
   }
   return true;
}

/**
 * Our sampling function that is executed by the spawned sample thread.
 * This function is declared as a static member of MyButtonDevice.  It simply
 * calls MyButtonDevice::sample() over and over.
 */
void MyButtonDevice::threadedSampleFunction(void* classPointer)
{
   MyButtonDevice* this_ptr = static_cast<MyButtonDevice*>( classPointer );

   // spin until someone kills "mSampleThread"
   while ( this_ptr->mRunning )
   {
     this_ptr->sample();
     vpr::System::sleep(1); //specify some time here, so you don't waste CPU cycles
   }
}
<|endoftext|>


<commit_before>/*
The MIT License (MIT)
Copyright (c) 2015 University of Central Florida's Computer Software Engineering
Scalable & Secure Systems (CSE - S3) Lab
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

#include "../libmem.h"

/*
 * libmem_test.cc
 *
 *  Created on: Oct 19, 2015
 *  @author Mike McGee
 */

//Main method that will test the execution of the libmem api
int main (int argc, char**argv) {
  // Use libmem to send a message to the server.

  //1. Connect to server [Test server is port 3490]
  int sock_fd = libmem::Connect(3490, "localhost", "");
  std::cout << "Connected" << std::endl;
  //2. Execute a SQLCommand
  libmem::ResultSet result_set = libmem::ExecuteSQL("SELECT * FROM Users",
                                                     sock_fd);

  // Try to print some of the result.
/*for (auto&& res: result_set) {
    for (auto&& pair: res) {
      std::cout << pair.get_name() << ":" << pair.get_value();
    }
    std::cout << std::endl;
  }*/

  std::cout << "Executed command" << std::endl;
  //3.Disconnect from database.
  libmem::Disconnect(sock_fd);
  std::cout << "Disconnected" << std::endl;

  return 0;
}

<commit_msg>Fixed loops in libmem_test. Realize I want to change whole thing now<commit_after>/*
The MIT License (MIT)
Copyright (c) 2015 University of Central Florida's Computer Software Engineering
Scalable & Secure Systems (CSE - S3) Lab
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

#include "../libmem.h"

/*
 * libmem_test.cc
 *
 *  Created on: Oct 19, 2015
 *  @author Mike McGee
 */

//Main method that will test the execution of the libmem api
int main (int argc, char**argv) {
  // Use libmem to send a message to the server.

  //1. Connect to server [Test server is port 3490]
  int sock_fd = libmem::Connect(3490, "localhost", "");
  std::cout << "Connected" << std::endl;
  //2. Execute a SQLCommand
  libmem::ResultSet result_set = libmem::ExecuteSQL("SELECT * FROM Users",
                                                     sock_fd);

  for (int i = 0, j = result_set.get_size(); i < j; ++i) {
    libmem::ResultRow current_row = result_set.get_next_row();
    std::vector<libmem::ResultPair> cur_vec = current_row.m_result_row;
    for (int k = 0, n = current_row.m_result_row.size(); k < n; ++k){
      std::cout << "Name :" << cur_vec.at(k).get_name();
      std::cout << "Value: " << cur_vec.at(k).get_value() << std::endl;
    }
  }

  std::cout << "Executed command" << std::endl;
  //3.Disconnect from database.
  libmem::Disconnect(sock_fd);
  std::cout << "Disconnected" << std::endl;

  return 0;
}

<|endoftext|>


<commit_before>//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
//  This source code is licensed under the BSD-style license found in the
//  LICENSE file in the root directory of this source tree. An additional grant
//  of patent rights can be found in the PATENTS file in the same directory.

#include <map>
#include <memory>
#include <string>

#include "db/db_impl.h"
#include "db/dbformat.h"
#include "db/table_properties_collector.h"
#include "rocksdb/table.h"
#include "table/block_based_table_factory.h"
#include "table/meta_blocks.h"
#include "table/plain_table_factory.h"
#include "table/table_builder.h"
#include "util/coding.h"
#include "util/testharness.h"
#include "util/testutil.h"

namespace rocksdb {

class TablePropertiesTest {
};

// TODO(kailiu) the following classes should be moved to some more general
// places, so that other tests can also make use of them.
// `FakeWritableFile` and `FakeRandomeAccessFile` bypass the real file system
// and therefore enable us to quickly setup the tests.
class FakeWritableFile : public WritableFile {
 public:
  ~FakeWritableFile() { }

  const std::string& contents() const { return contents_; }

  virtual Status Close() { return Status::OK(); }
  virtual Status Flush() { return Status::OK(); }
  virtual Status Sync() { return Status::OK(); }

  virtual Status Append(const Slice& data) {
    contents_.append(data.data(), data.size());
    return Status::OK();
  }

 private:
  std::string contents_;
};

class FakeRandomeAccessFile : public RandomAccessFile {
 public:
  explicit FakeRandomeAccessFile(const Slice& contents)
      : contents_(contents.data(), contents.size()) {
  }

  virtual ~FakeRandomeAccessFile() { }

  uint64_t Size() const { return contents_.size(); }

  virtual Status Read(uint64_t offset, size_t n, Slice* result,
                       char* scratch) const {
    if (offset > contents_.size()) {
      return Status::InvalidArgument("invalid Read offset");
    }
    if (offset + n > contents_.size()) {
      n = contents_.size() - offset;
    }
    memcpy(scratch, &contents_[offset], n);
    *result = Slice(scratch, n);
    return Status::OK();
  }

 private:
  std::string contents_;
};

class DumbLogger : public Logger {
 public:
  virtual void Logv(const char* format, va_list ap) { }
  virtual size_t GetLogFileSize() const { return 0; }
};

// Utilities test functions
void MakeBuilder(const Options& options,
                 const InternalKeyComparator& internal_comparator,
                 std::unique_ptr<FakeWritableFile>* writable,
                 std::unique_ptr<TableBuilder>* builder) {
  writable->reset(new FakeWritableFile);
  builder->reset(options.table_factory->NewTableBuilder(
      options, internal_comparator, writable->get(), options.compression));
}

// Collects keys that starts with "A" in a table.
class RegularKeysStartWithA: public TablePropertiesCollector {
 public:
   const char* Name() const { return "RegularKeysStartWithA"; }

   Status Finish(UserCollectedProperties* properties) {
     std::string encoded;
     PutVarint32(&encoded, count_);
     *properties = UserCollectedProperties {
       { "TablePropertiesTest", "Rocksdb" },
       { "Count", encoded }
     };
     return Status::OK();
   }

   Status Add(const Slice& user_key, const Slice& value) {
     // simply asssume all user keys are not empty.
     if (user_key.data()[0] == 'A') {
       ++count_;
     }
     return Status::OK();
   }

  virtual UserCollectedProperties GetReadableProperties() const {
      return {};
  }

 private:
  uint32_t count_ = 0;
};

extern uint64_t kBlockBasedTableMagicNumber;
extern uint64_t kPlainTableMagicNumber;
void TestCustomizedTablePropertiesCollector(
    uint64_t magic_number, bool encode_as_internal, const Options& options,
    const InternalKeyComparator& internal_comparator) {
  // make sure the entries will be inserted with order.
  std::map<std::string, std::string> kvs = {
    {"About   ", "val5"},  // starts with 'A'
    {"Abstract", "val2"},  // starts with 'A'
    {"Around  ", "val7"},  // starts with 'A'
    {"Beyond  ", "val3"},
    {"Builder ", "val1"},
    {"Cancel  ", "val4"},
    {"Find    ", "val6"},
  };

  // -- Step 1: build table
  std::unique_ptr<TableBuilder> builder;
  std::unique_ptr<FakeWritableFile> writable;
  MakeBuilder(options, internal_comparator, &writable, &builder);

  for (const auto& kv : kvs) {
    if (encode_as_internal) {
      InternalKey ikey(kv.first, 0, ValueType::kTypeValue);
      builder->Add(ikey.Encode(), kv.second);
    } else {
      builder->Add(kv.first, kv.second);
    }
  }
  ASSERT_OK(builder->Finish());

  // -- Step 2: Read properties
  FakeRandomeAccessFile readable(writable->contents());
  TableProperties* props;
  Status s = ReadTableProperties(
      &readable,
      writable->contents().size(),
      magic_number,
      Env::Default(),
      nullptr,
      &props
  );
  std::unique_ptr<TableProperties> props_guard(props);
  ASSERT_OK(s);

  auto user_collected = props->user_collected_properties;

  ASSERT_EQ("Rocksdb", user_collected.at("TablePropertiesTest"));

  uint32_t starts_with_A = 0;
  Slice key(user_collected.at("Count"));
  ASSERT_TRUE(GetVarint32(&key, &starts_with_A));
  ASSERT_EQ(3u, starts_with_A);
}

TEST(TablePropertiesTest, CustomizedTablePropertiesCollector) {
  // Test properties collectors with internal keys or regular keys
  // for block based table
  for (bool encode_as_internal : { true, false }) {
    Options options;
    auto collector = new RegularKeysStartWithA();
    if (encode_as_internal) {
      options.table_properties_collectors = {
        std::make_shared<UserKeyTablePropertiesCollector>(collector)
      };
    } else {
      options.table_properties_collectors.resize(1);
      options.table_properties_collectors[0].reset(collector);
    }
    test::PlainInternalKeyComparator ikc(options.comparator);
    TestCustomizedTablePropertiesCollector(kBlockBasedTableMagicNumber,
                                           encode_as_internal, options, ikc);
  }

  // test plain table
  Options options;
  options.table_properties_collectors.push_back(
      std::make_shared<RegularKeysStartWithA>()
  );
  options.table_factory = std::make_shared<PlainTableFactory>(8, 8, 0);
  test::PlainInternalKeyComparator ikc(options.comparator);
  TestCustomizedTablePropertiesCollector(kPlainTableMagicNumber, true, options,
                                         ikc);
}

void TestInternalKeyPropertiesCollector(
    uint64_t magic_number,
    bool sanitized,
    std::shared_ptr<TableFactory> table_factory) {
  InternalKey keys[] = {
    InternalKey("A       ", 0, ValueType::kTypeValue),
    InternalKey("B       ", 0, ValueType::kTypeValue),
    InternalKey("C       ", 0, ValueType::kTypeValue),
    InternalKey("W       ", 0, ValueType::kTypeDeletion),
    InternalKey("X       ", 0, ValueType::kTypeDeletion),
    InternalKey("Y       ", 0, ValueType::kTypeDeletion),
    InternalKey("Z       ", 0, ValueType::kTypeDeletion),
  };

  std::unique_ptr<TableBuilder> builder;
  std::unique_ptr<FakeWritableFile> writable;
  Options options;
  test::PlainInternalKeyComparator pikc(options.comparator);

  options.table_factory = table_factory;
  if (sanitized) {
    options.table_properties_collectors = {
      std::make_shared<RegularKeysStartWithA>()
    };
    // with sanitization, even regular properties collector will be able to
    // handle internal keys.
    auto comparator = options.comparator;
    // HACK: Set options.info_log to avoid writing log in
    // SanitizeOptions().
    options.info_log = std::make_shared<DumbLogger>();
    options = SanitizeOptions("db",            // just a place holder
                              &pikc, nullptr,  // don't care filter policy
                              options);
    options.comparator = comparator;
  } else {
    options.table_properties_collectors = {
      std::make_shared<InternalKeyPropertiesCollector>()
    };
  }

  MakeBuilder(options, pikc, &writable, &builder);
  for (const auto& k : keys) {
    builder->Add(k.Encode(), "val");
  }

  ASSERT_OK(builder->Finish());

  FakeRandomeAccessFile readable(writable->contents());
  TableProperties* props;
  Status s = ReadTableProperties(
      &readable,
      writable->contents().size(),
      magic_number,
      Env::Default(),
      nullptr,
      &props
  );
  ASSERT_OK(s);

  std::unique_ptr<TableProperties> props_guard(props);
  auto user_collected = props->user_collected_properties;
  uint64_t deleted = GetDeletedKeys(user_collected);
  ASSERT_EQ(4u, deleted);

  if (sanitized) {
    uint32_t starts_with_A = 0;
    Slice key(user_collected.at("Count"));
    ASSERT_TRUE(GetVarint32(&key, &starts_with_A));
    ASSERT_EQ(1u, starts_with_A);
  }
}

TEST(TablePropertiesTest, InternalKeyPropertiesCollector) {
  TestInternalKeyPropertiesCollector(
      kBlockBasedTableMagicNumber,
      true /* sanitize */,
      std::make_shared<BlockBasedTableFactory>()
  );
  TestInternalKeyPropertiesCollector(
      kBlockBasedTableMagicNumber,
      true /* not sanitize */,
      std::make_shared<BlockBasedTableFactory>()
  );
  TestInternalKeyPropertiesCollector(
      kPlainTableMagicNumber,
      false /* not sanitize */,
      std::make_shared<PlainTableFactory>(8, 8, 0)
  );
}

}  // namespace rocksdb

int main(int argc, char** argv) {
  return rocksdb::test::RunAllTests();
}
<commit_msg>Fix problem 3 for issue #80<commit_after>//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
//  This source code is licensed under the BSD-style license found in the
//  LICENSE file in the root directory of this source tree. An additional grant
//  of patent rights can be found in the PATENTS file in the same directory.

#include <map>
#include <memory>
#include <string>

#include "db/db_impl.h"
#include "db/dbformat.h"
#include "db/table_properties_collector.h"
#include "rocksdb/table.h"
#include "table/block_based_table_factory.h"
#include "table/meta_blocks.h"
#include "table/plain_table_factory.h"
#include "table/table_builder.h"
#include "util/coding.h"
#include "util/testharness.h"
#include "util/testutil.h"

namespace rocksdb {

class TablePropertiesTest {
};

// TODO(kailiu) the following classes should be moved to some more general
// places, so that other tests can also make use of them.
// `FakeWritableFile` and `FakeRandomeAccessFile` bypass the real file system
// and therefore enable us to quickly setup the tests.
class FakeWritableFile : public WritableFile {
 public:
  ~FakeWritableFile() { }

  const std::string& contents() const { return contents_; }

  virtual Status Close() { return Status::OK(); }
  virtual Status Flush() { return Status::OK(); }
  virtual Status Sync() { return Status::OK(); }

  virtual Status Append(const Slice& data) {
    contents_.append(data.data(), data.size());
    return Status::OK();
  }

 private:
  std::string contents_;
};

class FakeRandomeAccessFile : public RandomAccessFile {
 public:
  explicit FakeRandomeAccessFile(const Slice& contents)
      : contents_(contents.data(), contents.size()) {
  }

  virtual ~FakeRandomeAccessFile() { }

  uint64_t Size() const { return contents_.size(); }

  virtual Status Read(uint64_t offset, size_t n, Slice* result,
                       char* scratch) const {
    if (offset > contents_.size()) {
      return Status::InvalidArgument("invalid Read offset");
    }
    if (offset + n > contents_.size()) {
      n = contents_.size() - offset;
    }
    memcpy(scratch, &contents_[offset], n);
    *result = Slice(scratch, n);
    return Status::OK();
  }

 private:
  std::string contents_;
};

class DumbLogger : public Logger {
 public:
  virtual void Logv(const char* format, va_list ap) { }
  virtual size_t GetLogFileSize() const { return 0; }
};

// Utilities test functions
void MakeBuilder(const Options& options,
                 const InternalKeyComparator& internal_comparator,
                 std::unique_ptr<FakeWritableFile>* writable,
                 std::unique_ptr<TableBuilder>* builder) {
  writable->reset(new FakeWritableFile);
  builder->reset(options.table_factory->NewTableBuilder(
      options, internal_comparator, writable->get(), options.compression));
}

// Collects keys that starts with "A" in a table.
class RegularKeysStartWithA: public TablePropertiesCollector {
 public:
   const char* Name() const { return "RegularKeysStartWithA"; }

   Status Finish(UserCollectedProperties* properties) {
     std::string encoded;
     PutVarint32(&encoded, count_);
     *properties = UserCollectedProperties {
       { "TablePropertiesTest", "Rocksdb" },
       { "Count", encoded }
     };
     return Status::OK();
   }

   Status Add(const Slice& user_key, const Slice& value) {
     // simply asssume all user keys are not empty.
     if (user_key.data()[0] == 'A') {
       ++count_;
     }
     return Status::OK();
   }

  virtual UserCollectedProperties GetReadableProperties() const {
    return UserCollectedProperties{};
  }

 private:
  uint32_t count_ = 0;
};

extern uint64_t kBlockBasedTableMagicNumber;
extern uint64_t kPlainTableMagicNumber;
void TestCustomizedTablePropertiesCollector(
    uint64_t magic_number, bool encode_as_internal, const Options& options,
    const InternalKeyComparator& internal_comparator) {
  // make sure the entries will be inserted with order.
  std::map<std::string, std::string> kvs = {
    {"About   ", "val5"},  // starts with 'A'
    {"Abstract", "val2"},  // starts with 'A'
    {"Around  ", "val7"},  // starts with 'A'
    {"Beyond  ", "val3"},
    {"Builder ", "val1"},
    {"Cancel  ", "val4"},
    {"Find    ", "val6"},
  };

  // -- Step 1: build table
  std::unique_ptr<TableBuilder> builder;
  std::unique_ptr<FakeWritableFile> writable;
  MakeBuilder(options, internal_comparator, &writable, &builder);

  for (const auto& kv : kvs) {
    if (encode_as_internal) {
      InternalKey ikey(kv.first, 0, ValueType::kTypeValue);
      builder->Add(ikey.Encode(), kv.second);
    } else {
      builder->Add(kv.first, kv.second);
    }
  }
  ASSERT_OK(builder->Finish());

  // -- Step 2: Read properties
  FakeRandomeAccessFile readable(writable->contents());
  TableProperties* props;
  Status s = ReadTableProperties(
      &readable,
      writable->contents().size(),
      magic_number,
      Env::Default(),
      nullptr,
      &props
  );
  std::unique_ptr<TableProperties> props_guard(props);
  ASSERT_OK(s);

  auto user_collected = props->user_collected_properties;

  ASSERT_EQ("Rocksdb", user_collected.at("TablePropertiesTest"));

  uint32_t starts_with_A = 0;
  Slice key(user_collected.at("Count"));
  ASSERT_TRUE(GetVarint32(&key, &starts_with_A));
  ASSERT_EQ(3u, starts_with_A);
}

TEST(TablePropertiesTest, CustomizedTablePropertiesCollector) {
  // Test properties collectors with internal keys or regular keys
  // for block based table
  for (bool encode_as_internal : { true, false }) {
    Options options;
    auto collector = new RegularKeysStartWithA();
    if (encode_as_internal) {
      options.table_properties_collectors = {
        std::make_shared<UserKeyTablePropertiesCollector>(collector)
      };
    } else {
      options.table_properties_collectors.resize(1);
      options.table_properties_collectors[0].reset(collector);
    }
    test::PlainInternalKeyComparator ikc(options.comparator);
    TestCustomizedTablePropertiesCollector(kBlockBasedTableMagicNumber,
                                           encode_as_internal, options, ikc);
  }

  // test plain table
  Options options;
  options.table_properties_collectors.push_back(
      std::make_shared<RegularKeysStartWithA>()
  );
  options.table_factory = std::make_shared<PlainTableFactory>(8, 8, 0);
  test::PlainInternalKeyComparator ikc(options.comparator);
  TestCustomizedTablePropertiesCollector(kPlainTableMagicNumber, true, options,
                                         ikc);
}

void TestInternalKeyPropertiesCollector(
    uint64_t magic_number,
    bool sanitized,
    std::shared_ptr<TableFactory> table_factory) {
  InternalKey keys[] = {
    InternalKey("A       ", 0, ValueType::kTypeValue),
    InternalKey("B       ", 0, ValueType::kTypeValue),
    InternalKey("C       ", 0, ValueType::kTypeValue),
    InternalKey("W       ", 0, ValueType::kTypeDeletion),
    InternalKey("X       ", 0, ValueType::kTypeDeletion),
    InternalKey("Y       ", 0, ValueType::kTypeDeletion),
    InternalKey("Z       ", 0, ValueType::kTypeDeletion),
  };

  std::unique_ptr<TableBuilder> builder;
  std::unique_ptr<FakeWritableFile> writable;
  Options options;
  test::PlainInternalKeyComparator pikc(options.comparator);

  options.table_factory = table_factory;
  if (sanitized) {
    options.table_properties_collectors = {
      std::make_shared<RegularKeysStartWithA>()
    };
    // with sanitization, even regular properties collector will be able to
    // handle internal keys.
    auto comparator = options.comparator;
    // HACK: Set options.info_log to avoid writing log in
    // SanitizeOptions().
    options.info_log = std::make_shared<DumbLogger>();
    options = SanitizeOptions("db",            // just a place holder
                              &pikc, nullptr,  // don't care filter policy
                              options);
    options.comparator = comparator;
  } else {
    options.table_properties_collectors = {
      std::make_shared<InternalKeyPropertiesCollector>()
    };
  }

  MakeBuilder(options, pikc, &writable, &builder);
  for (const auto& k : keys) {
    builder->Add(k.Encode(), "val");
  }

  ASSERT_OK(builder->Finish());

  FakeRandomeAccessFile readable(writable->contents());
  TableProperties* props;
  Status s = ReadTableProperties(
      &readable,
      writable->contents().size(),
      magic_number,
      Env::Default(),
      nullptr,
      &props
  );
  ASSERT_OK(s);

  std::unique_ptr<TableProperties> props_guard(props);
  auto user_collected = props->user_collected_properties;
  uint64_t deleted = GetDeletedKeys(user_collected);
  ASSERT_EQ(4u, deleted);

  if (sanitized) {
    uint32_t starts_with_A = 0;
    Slice key(user_collected.at("Count"));
    ASSERT_TRUE(GetVarint32(&key, &starts_with_A));
    ASSERT_EQ(1u, starts_with_A);
  }
}

TEST(TablePropertiesTest, InternalKeyPropertiesCollector) {
  TestInternalKeyPropertiesCollector(
      kBlockBasedTableMagicNumber,
      true /* sanitize */,
      std::make_shared<BlockBasedTableFactory>()
  );
  TestInternalKeyPropertiesCollector(
      kBlockBasedTableMagicNumber,
      true /* not sanitize */,
      std::make_shared<BlockBasedTableFactory>()
  );
  TestInternalKeyPropertiesCollector(
      kPlainTableMagicNumber,
      false /* not sanitize */,
      std::make_shared<PlainTableFactory>(8, 8, 0)
  );
}

}  // namespace rocksdb

int main(int argc, char** argv) {
  return rocksdb::test::RunAllTests();
}
<|endoftext|>