<commit_before>// -*- coding: utf-8 -*-
// Copyright (C) 2012-2014 MUJIN Inc. <rosen.diankov@mujin.co.jp>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "common.h"
#include "controllerclientimpl.h"
#include "binpickingtaskzmq.h"
#include <boost/property_tree/json_parser.hpp>
#include <boost/property_tree/exceptions.hpp>
#include "mujincontrollerclient/mujinzmq.h"

#include <algorithm> // find

#include "logging.h"

MUJIN_LOGGER("mujin.controllerclientcpp.binpickingtask.zmq");

using namespace mujinzmq;

namespace mujinclient {
using namespace utils;

class ZmqMujinControllerClient : public mujinzmq::ZmqClient
{

public:
    ZmqMujinControllerClient(boost::shared_ptr<zmq::context_t> context, const std::string& host, const int port);

    virtual ~ZmqMujinControllerClient();

};

ZmqMujinControllerClient::ZmqMujinControllerClient(boost::shared_ptr<zmq::context_t> context, const std::string& host, const int port) : ZmqClient(host, port)
{
    _InitializeSocket(context);
}

ZmqMujinControllerClient::~ZmqMujinControllerClient()
{
    // _DestroySocket() is called in  ~ZmqClient()
}

BinPickingTaskZmqResource::BinPickingTaskZmqResource(ControllerClientPtr c, const std::string& pk, const std::string& scenepk, const std::string& tasktype) : BinPickingTaskResource(c, pk, scenepk, tasktype)
{
}

BinPickingTaskZmqResource::~BinPickingTaskZmqResource()
{
}

void BinPickingTaskZmqResource::Initialize(const std::string& defaultTaskParameters,  const int zmqPort, const int heartbeatPort, boost::shared_ptr<zmq::context_t> zmqcontext, const bool initializezmq, const double reinitializetimeout, const double timeout, const std::string& userinfo, const std::string& slaverequestid)
{
    BinPickingTaskResource::Initialize(defaultTaskParameters, zmqPort, heartbeatPort, zmqcontext, initializezmq, reinitializetimeout, timeout, userinfo, slaverequestid);

    if (initializezmq) {
        InitializeZMQ(reinitializetimeout, timeout);
    }

    _zmqmujincontrollerclient.reset(new ZmqMujinControllerClient(_zmqcontext, _mujinControllerIp, _zmqPort));
    if (!_zmqmujincontrollerclient) {
        throw MujinException(boost::str(boost::format("Failed to establish ZMQ connection to mujin controller at %s:%d")%_mujinControllerIp%_zmqPort), MEC_Failed);
    }
    if (!_pHeartbeatMonitorThread) {
        _bShutdownHeartbeatMonitor = false;
        if (reinitializetimeout > 0 ) {
            _pHeartbeatMonitorThread.reset(new boost::thread(boost::bind(&BinPickingTaskZmqResource::_HeartbeatMonitorThread, this, reinitializetimeout, timeout)));
        }
    }
}

boost::property_tree::ptree BinPickingTaskZmqResource::ExecuteCommand(const std::string& taskparameters, const double timeout /* [sec] */, const bool getresult)
{
    if (!_bIsInitialized) {
        throw MujinException("BinPicking task is not initialized, please call Initialzie() first.", MEC_Failed);
    }

    if (!_zmqmujincontrollerclient) {
        MUJIN_LOG_ERROR("zmqcontrollerclient is not initialized! initialize");
        _zmqmujincontrollerclient.reset(new ZmqMujinControllerClient(_zmqcontext, _mujinControllerIp, _zmqPort));
    }

    std::stringstream ss;
    ss << "{\"fnname\": \"";
    if (_tasktype == "binpicking") {
        ss << "binpicking.";
    }
    ss << "RunCommand\", \"taskparams\": {\"tasktype\": \"" << _tasktype << "\", ";

    ss << "\"taskparameters\": " << taskparameters << ", ";
    ss << "\"sceneparams\": " << _sceneparams_json << "}, ";
    ss << "\"userinfo\": " << _userinfo_json;
    if (_slaverequestid != "") {
        ss << ", " << GetJsonString("slaverequestid", _slaverequestid);
    }
    ss << "}";
    std::string command = ss.str();
    //    std::cout << "Sending " << command << " from " << __func__ << std::endl;

    //std::string task = "{\"tasktype\": \"binpicking\", \"taskparameters\": " + command + "}";
    boost::property_tree::ptree pt;
    if (getresult) {
        std::stringstream result_ss;

        try{
            result_ss << _zmqmujincontrollerclient->Call(command, timeout);
        }
        catch (const MujinException& e) {
            MUJIN_LOG_ERROR(e.what());
            if (e.GetCode() == MEC_ZMQNoResponse) {
                MUJIN_LOG_INFO("reinitializing zmq connection with the slave");
                _zmqmujincontrollerclient.reset(new ZmqMujinControllerClient(_zmqcontext, _mujinControllerIp, _zmqPort));
                if (!_zmqmujincontrollerclient) {
                    throw MujinException(boost::str(boost::format("Failed to establish ZMQ connection to mujin controller at %s:%d")%_mujinControllerIp%_zmqPort), MEC_Failed);
                }
            }
        }

        //std::cout << result_ss.str() << std::endl;
        try {
            boost::property_tree::read_json(result_ss, pt);
            boost::property_tree::ptree::const_assoc_iterator iterror = pt.find("error");
            if( iterror != pt.not_found() ) {
                boost::optional<std::string> erroroptional = iterror->second.get_optional<std::string>("errorcode");
                boost::optional<std::string> descriptionoptional = iterror->second.get_optional<std::string>("description");
                if (!!erroroptional && erroroptional.get().size() > 0 ) {
                    std::string serror;
                    if (!!descriptionoptional && descriptionoptional.get().size() > 0 ) {
                        serror = descriptionoptional.get();
                    }
                    else {
                        serror = erroroptional.get();
                    }
                    if( serror.size() > 1000 ) {
                        MUJIN_LOG_ERROR(str(boost::format("truncated original error message from %d")%serror.size()));
                        serror = serror.substr(0,1000);
                    }

                    throw MujinException(str(boost::format("Error when calling binpicking.RunCommand: %s")%serror), MEC_BinPickingError);
                }
            }
        } catch (boost::property_tree::ptree_error& e) {
            throw MujinException(boost::str(boost::format("Failed to parse json which is received from mujin controller: \nreceived message: %s \nerror message: %s")%result_ss.str()%e.what()), MEC_Failed);
        }
    } else {
        try{
            _zmqmujincontrollerclient->Call(command, timeout);
            // TODO since not getting response, internal zmq will be in a bad state, have to recreate
        }
        catch (const MujinException& e) {
            MUJIN_LOG_ERROR(e.what());
            if (e.GetCode() == MEC_ZMQNoResponse) {
                MUJIN_LOG_INFO("reinitializing zmq connection with the slave");
                _zmqmujincontrollerclient.reset(new ZmqMujinControllerClient(_zmqcontext, _mujinControllerIp, _zmqPort));
                if (!_zmqmujincontrollerclient) {
                    throw MujinException(boost::str(boost::format("Failed to establish ZMQ connection to mujin controller at %s:%d")%_mujinControllerIp%_zmqPort), MEC_Failed);
                }
            }
        }
    }
    return pt;
}

void BinPickingTaskZmqResource::InitializeZMQ(const double reinitializetimeout, const double timeout)
{
}

void BinPickingTaskZmqResource::_HeartbeatMonitorThread(const double reinitializetimeout, const double commandtimeout)
{
    boost::shared_ptr<zmq::socket_t>  socket;
    boost::property_tree::ptree pt;
    BinPickingTaskResource::ResultHeartBeat heartbeat;
    heartbeat._slaverequestid = _slaverequestid;
    while (!_bShutdownHeartbeatMonitor) {
        if (!!socket) {
            socket->close();
            socket.reset();
        }
        socket.reset(new zmq::socket_t((*_zmqcontext.get()),ZMQ_SUB));
        std::stringstream ss;
        ss << _heartbeatPort;
        socket->connect (("tcp://"+ _mujinControllerIp+":"+ss.str()).c_str());
        socket->setsockopt(ZMQ_SUBSCRIBE, "", 0);

        zmq::pollitem_t pollitem;
        memset(&pollitem, 0, sizeof(zmq::pollitem_t));
        pollitem.socket = socket->operator void*();
        pollitem.events = ZMQ_POLLIN;
        unsigned long long lastheartbeat = GetMilliTime();
        while (!_bShutdownHeartbeatMonitor && (GetMilliTime() - lastheartbeat) / 1000.0f < reinitializetimeout) {
            zmq::poll(&pollitem,1, 50); // wait 50 ms for message
            if (pollitem.revents & ZMQ_POLLIN) {
                zmq::message_t reply;
                socket->recv(&reply);
                std::string replystring((char *) reply.data (), (size_t) reply.size());
#ifdef _WIN32
                // sometimes buffer can container \n or \\, which windows boost property_tree does not like
                std::string newbuffer;
                std::vector< std::pair<std::string, std::string> > serachpairs(1);
                serachpairs[0].first = "\\"; serachpairs[0].second = "";
                SearchAndReplace(newbuffer, replystring, serachpairs);
                std::stringstream replystring_ss(newbuffer);
#else
                std::stringstream replystring_ss(replystring);
#endif
                try{
                    boost::property_tree::read_json(replystring_ss, pt);
                }
                catch (std::exception const &e) {
                    MUJIN_LOG_ERROR("HeartBeat reply is not JSON");
                    MUJIN_LOG_ERROR(e.what());
                    continue;
                }
                heartbeat.Parse(pt);
                {
                    boost::mutex::scoped_lock lock(_mutexTaskState);
                    _taskstate = heartbeat.taskstate;
                }
                //BINPICKING_LOG_ERROR(replystring);

                if (heartbeat.status != std::string("lost") && heartbeat.status.size() > 1) {
                    lastheartbeat = GetMilliTime();
                }
            }
        }
        if (!_bShutdownHeartbeatMonitor) {
            std::stringstream ss;
            ss << (double)((GetMilliTime() - lastheartbeat)/1000.0f) << " seconds passed since last heartbeat signal, re-intializing ZMQ server.";
            MUJIN_LOG_INFO(ss.str());
        }
    }
}

} // end namespace mujinclient
<commit_msg>throw when timeout<commit_after>// -*- coding: utf-8 -*-
// Copyright (C) 2012-2014 MUJIN Inc. <rosen.diankov@mujin.co.jp>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "common.h"
#include "controllerclientimpl.h"
#include "binpickingtaskzmq.h"
#include <boost/property_tree/json_parser.hpp>
#include <boost/property_tree/exceptions.hpp>
#include "mujincontrollerclient/mujinzmq.h"

#include <algorithm> // find

#include "logging.h"

MUJIN_LOGGER("mujin.controllerclientcpp.binpickingtask.zmq");

using namespace mujinzmq;

namespace mujinclient {
using namespace utils;

class ZmqMujinControllerClient : public mujinzmq::ZmqClient
{

public:
    ZmqMujinControllerClient(boost::shared_ptr<zmq::context_t> context, const std::string& host, const int port);

    virtual ~ZmqMujinControllerClient();

};

ZmqMujinControllerClient::ZmqMujinControllerClient(boost::shared_ptr<zmq::context_t> context, const std::string& host, const int port) : ZmqClient(host, port)
{
    _InitializeSocket(context);
}

ZmqMujinControllerClient::~ZmqMujinControllerClient()
{
    // _DestroySocket() is called in  ~ZmqClient()
}

BinPickingTaskZmqResource::BinPickingTaskZmqResource(ControllerClientPtr c, const std::string& pk, const std::string& scenepk, const std::string& tasktype) : BinPickingTaskResource(c, pk, scenepk, tasktype)
{
}

BinPickingTaskZmqResource::~BinPickingTaskZmqResource()
{
}

void BinPickingTaskZmqResource::Initialize(const std::string& defaultTaskParameters,  const int zmqPort, const int heartbeatPort, boost::shared_ptr<zmq::context_t> zmqcontext, const bool initializezmq, const double reinitializetimeout, const double timeout, const std::string& userinfo, const std::string& slaverequestid)
{
    BinPickingTaskResource::Initialize(defaultTaskParameters, zmqPort, heartbeatPort, zmqcontext, initializezmq, reinitializetimeout, timeout, userinfo, slaverequestid);

    if (initializezmq) {
        InitializeZMQ(reinitializetimeout, timeout);
    }

    _zmqmujincontrollerclient.reset(new ZmqMujinControllerClient(_zmqcontext, _mujinControllerIp, _zmqPort));
    if (!_zmqmujincontrollerclient) {
        throw MujinException(boost::str(boost::format("Failed to establish ZMQ connection to mujin controller at %s:%d")%_mujinControllerIp%_zmqPort), MEC_Failed);
    }
    if (!_pHeartbeatMonitorThread) {
        _bShutdownHeartbeatMonitor = false;
        if (reinitializetimeout > 0 ) {
            _pHeartbeatMonitorThread.reset(new boost::thread(boost::bind(&BinPickingTaskZmqResource::_HeartbeatMonitorThread, this, reinitializetimeout, timeout)));
        }
    }
}

boost::property_tree::ptree BinPickingTaskZmqResource::ExecuteCommand(const std::string& taskparameters, const double timeout /* [sec] */, const bool getresult)
{
    if (!_bIsInitialized) {
        throw MujinException("BinPicking task is not initialized, please call Initialzie() first.", MEC_Failed);
    }

    if (!_zmqmujincontrollerclient) {
        MUJIN_LOG_ERROR("zmqcontrollerclient is not initialized! initialize");
        _zmqmujincontrollerclient.reset(new ZmqMujinControllerClient(_zmqcontext, _mujinControllerIp, _zmqPort));
    }

    std::stringstream ss;
    ss << "{\"fnname\": \"";
    if (_tasktype == "binpicking") {
        ss << "binpicking.";
    }
    ss << "RunCommand\", \"taskparams\": {\"tasktype\": \"" << _tasktype << "\", ";

    ss << "\"taskparameters\": " << taskparameters << ", ";
    ss << "\"sceneparams\": " << _sceneparams_json << "}, ";
    ss << "\"userinfo\": " << _userinfo_json;
    if (_slaverequestid != "") {
        ss << ", " << GetJsonString("slaverequestid", _slaverequestid);
    }
    ss << "}";
    std::string command = ss.str();
    //    std::cout << "Sending " << command << " from " << __func__ << std::endl;

    //std::string task = "{\"tasktype\": \"binpicking\", \"taskparameters\": " + command + "}";
    boost::property_tree::ptree pt;
    if (getresult) {
        std::stringstream result_ss;

        try{
            result_ss << _zmqmujincontrollerclient->Call(command, timeout);
        }
        catch (const MujinException& e) {
            MUJIN_LOG_ERROR(e.what());
            if (e.GetCode() == MEC_ZMQNoResponse) {
                MUJIN_LOG_INFO("reinitializing zmq connection with the slave");
                _zmqmujincontrollerclient.reset(new ZmqMujinControllerClient(_zmqcontext, _mujinControllerIp, _zmqPort));
                if (!_zmqmujincontrollerclient) {
                    throw MujinException(boost::str(boost::format("Failed to establish ZMQ connection to mujin controller at %s:%d")%_mujinControllerIp%_zmqPort), MEC_Failed);
                }
            } else if (e.GetCode() == MEC_Timeout) {
                std::string errstr;
                if (taskparameters.size()>1000) {
                    errstr = taskparameters.substr(0, 1000);
                } else {
                    errstr = taskparameters;
                }
                throw MujinException(boost::str(boost::format("Timed out receiving response of command with taskparameters=%s...")%errstr));
            }
        }

        //std::cout << result_ss.str() << std::endl;
        try {
            boost::property_tree::read_json(result_ss, pt);
            boost::property_tree::ptree::const_assoc_iterator iterror = pt.find("error");
            if( iterror != pt.not_found() ) {
                boost::optional<std::string> erroroptional = iterror->second.get_optional<std::string>("errorcode");
                boost::optional<std::string> descriptionoptional = iterror->second.get_optional<std::string>("description");
                if (!!erroroptional && erroroptional.get().size() > 0 ) {
                    std::string serror;
                    if (!!descriptionoptional && descriptionoptional.get().size() > 0 ) {
                        serror = descriptionoptional.get();
                    }
                    else {
                        serror = erroroptional.get();
                    }
                    if( serror.size() > 1000 ) {
                        MUJIN_LOG_ERROR(str(boost::format("truncated original error message from %d")%serror.size()));
                        serror = serror.substr(0,1000);
                    }

                    throw MujinException(str(boost::format("Error when calling binpicking.RunCommand: %s")%serror), MEC_BinPickingError);
                }
            }
        } catch (boost::property_tree::ptree_error& e) {
            throw MujinException(boost::str(boost::format("Failed to parse json which is received from mujin controller: \nreceived message: %s \nerror message: %s")%result_ss.str()%e.what()), MEC_Failed);
        }
    } else {
        try{
            _zmqmujincontrollerclient->Call(command, timeout);
            // TODO since not getting response, internal zmq will be in a bad state, have to recreate
        }
        catch (const MujinException& e) {
            MUJIN_LOG_ERROR(e.what());
            if (e.GetCode() == MEC_ZMQNoResponse) {
                MUJIN_LOG_INFO("reinitializing zmq connection with the slave");
                _zmqmujincontrollerclient.reset(new ZmqMujinControllerClient(_zmqcontext, _mujinControllerIp, _zmqPort));
                if (!_zmqmujincontrollerclient) {
                    throw MujinException(boost::str(boost::format("Failed to establish ZMQ connection to mujin controller at %s:%d")%_mujinControllerIp%_zmqPort), MEC_Failed);
                }
            } else if (e.GetCode() == MEC_Timeout) {
                std::string errstr;
                if (command.size()>1000) {
                    errstr = command.substr(0, 1000);
                } else {
                    errstr = command;
                }
                throw MujinException(boost::str(boost::format("Timed out receiving response of command with taskparameters=%s...")%errstr));
            }
        }
    }
    return pt;
}

void BinPickingTaskZmqResource::InitializeZMQ(const double reinitializetimeout, const double timeout)
{
}

void BinPickingTaskZmqResource::_HeartbeatMonitorThread(const double reinitializetimeout, const double commandtimeout)
{
    boost::shared_ptr<zmq::socket_t>  socket;
    boost::property_tree::ptree pt;
    BinPickingTaskResource::ResultHeartBeat heartbeat;
    heartbeat._slaverequestid = _slaverequestid;
    while (!_bShutdownHeartbeatMonitor) {
        if (!!socket) {
            socket->close();
            socket.reset();
        }
        socket.reset(new zmq::socket_t((*_zmqcontext.get()),ZMQ_SUB));
        std::stringstream ss;
        ss << _heartbeatPort;
        socket->connect (("tcp://"+ _mujinControllerIp+":"+ss.str()).c_str());
        socket->setsockopt(ZMQ_SUBSCRIBE, "", 0);

        zmq::pollitem_t pollitem;
        memset(&pollitem, 0, sizeof(zmq::pollitem_t));
        pollitem.socket = socket->operator void*();
        pollitem.events = ZMQ_POLLIN;
        unsigned long long lastheartbeat = GetMilliTime();
        while (!_bShutdownHeartbeatMonitor && (GetMilliTime() - lastheartbeat) / 1000.0f < reinitializetimeout) {
            zmq::poll(&pollitem,1, 50); // wait 50 ms for message
            if (pollitem.revents & ZMQ_POLLIN) {
                zmq::message_t reply;
                socket->recv(&reply);
                std::string replystring((char *) reply.data (), (size_t) reply.size());
#ifdef _WIN32
                // sometimes buffer can container \n or \\, which windows boost property_tree does not like
                std::string newbuffer;
                std::vector< std::pair<std::string, std::string> > serachpairs(1);
                serachpairs[0].first = "\\"; serachpairs[0].second = "";
                SearchAndReplace(newbuffer, replystring, serachpairs);
                std::stringstream replystring_ss(newbuffer);
#else
                std::stringstream replystring_ss(replystring);
#endif
                try{
                    boost::property_tree::read_json(replystring_ss, pt);
                }
                catch (std::exception const &e) {
                    MUJIN_LOG_ERROR("HeartBeat reply is not JSON");
                    MUJIN_LOG_ERROR(e.what());
                    continue;
                }
                heartbeat.Parse(pt);
                {
                    boost::mutex::scoped_lock lock(_mutexTaskState);
                    _taskstate = heartbeat.taskstate;
                }
                //BINPICKING_LOG_ERROR(replystring);

                if (heartbeat.status != std::string("lost") && heartbeat.status.size() > 1) {
                    lastheartbeat = GetMilliTime();
                }
            }
        }
        if (!_bShutdownHeartbeatMonitor) {
            std::stringstream ss;
            ss << (double)((GetMilliTime() - lastheartbeat)/1000.0f) << " seconds passed since last heartbeat signal, re-intializing ZMQ server.";
            MUJIN_LOG_INFO(ss.str());
        }
    }
}

} // end namespace mujinclient
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2018 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "src/traced/probes/probes_producer.h"

#include <stdio.h>
#include <string>

#include "perfetto/base/logging.h"
#include "perfetto/traced/traced.h"
#include "perfetto/tracing/core/data_source_config.h"
#include "perfetto/tracing/core/data_source_descriptor.h"
#include "perfetto/tracing/core/ftrace_config.h"
#include "perfetto/tracing/core/trace_config.h"
#include "perfetto/tracing/core/trace_packet.h"

#include "src/process_stats/file_utils.h"
#include "src/process_stats/procfs_utils.h"

#include "perfetto/trace/ftrace/ftrace_event_bundle.pbzero.h"
#include "perfetto/trace/ps/process_tree.pbzero.h"
#include "perfetto/trace/trace_packet.pbzero.h"

namespace perfetto {
namespace {

uint64_t kInitialConnectionBackoffMs = 100;
uint64_t kMaxConnectionBackoffMs = 30 * 1000;
const char* kFtraceSourceName = "com.google.perfetto.ftrace";
const char* kProcessStatsSourceName = "com.google.perfetto.process_stats";

}  // namespace.

// State transition diagram:
//                    +----------------------------+
//                    v                            +
// NotStarted -> NotConnected -> Connecting -> Connected
//                    ^              +
//                    +--------------+
//

ProbesProducer::~ProbesProducer() = default;

void ProbesProducer::OnConnect() {
  PERFETTO_DCHECK(state_ == kConnecting);
  state_ = kConnected;
  ResetConnectionBackoff();
  PERFETTO_LOG("Connected to the service");

  DataSourceDescriptor ftrace_descriptor;
  ftrace_descriptor.set_name(kFtraceSourceName);
  endpoint_->RegisterDataSource(ftrace_descriptor,
                                [](DataSourceInstanceID id) {});

  DataSourceDescriptor process_stats_descriptor;
  process_stats_descriptor.set_name(kProcessStatsSourceName);
  endpoint_->RegisterDataSource(process_stats_descriptor,
                                [](DataSourceInstanceID id) {});
}

void ProbesProducer::OnDisconnect() {
  PERFETTO_DCHECK(state_ == kConnected || state_ == kConnecting);
  state_ = kNotConnected;
  PERFETTO_LOG("Disconnected from tracing service");
  IncreaseConnectionBackoff();

  // TODO(hjd): Erase all sinks and add e2e test for this.
  task_runner_->PostDelayedTask([this] { this->Connect(); },
                                connection_backoff_ms_);
}

void ProbesProducer::CreateDataSourceInstance(
    DataSourceInstanceID id,
    const DataSourceConfig& source_config) {
  instances_[id] = source_config.name();
  // PERFETTO_LOG("DataSourceInstanceID: %llu", id);
  if (source_config.name() == kFtraceSourceName) {
    CreateFtraceDataSourceInstance(id, source_config);
  } else if (source_config.name() == kProcessStatsSourceName) {
    CreateProcessStatsDataSourceInstance(source_config);
  } else {
    PERFETTO_ELOG("Data source name: %s not recognised.",
                  source_config.name().c_str());
  }
}

void ProbesProducer::CreateFtraceDataSourceInstance(
    DataSourceInstanceID id,
    const DataSourceConfig& source_config) {
  // Don't retry if FtraceController::Create() failed once.
  // This can legitimately happen on user builds where we cannot access the
  // debug paths, e.g., because of SELinux rules.
  if (ftrace_creation_failed_)
    return;

  // Lazily create on the first instance.
  if (!ftrace_) {
    ftrace_ = FtraceController::Create(task_runner_);

    if (!ftrace_) {
      PERFETTO_ELOG("Failed to create FtraceController");
      ftrace_creation_failed_ = true;
      return;
    }

    ftrace_->DisableAllEvents();
    ftrace_->ClearTrace();
  }

  PERFETTO_LOG("Ftrace start (id=%" PRIu64 ", target_buf=%" PRIu32 ")", id,
               source_config.target_buffer());

  FtraceConfig proto_config = source_config.ftrace_config();

  // TODO(hjd): Static cast is bad, target_buffer() should return a BufferID.
  auto trace_writer = endpoint_->CreateTraceWriter(
      static_cast<BufferID>(source_config.target_buffer()));
  auto delegate =
      std::unique_ptr<SinkDelegate>(new SinkDelegate(std::move(trace_writer)));
  auto sink = ftrace_->CreateSink(std::move(proto_config), delegate.get());
  PERFETTO_CHECK(sink);
  delegate->sink(std::move(sink));
  delegates_.emplace(id, std::move(delegate));
  // Building on Android, watchdogs_.emplace(id, 2* source_config.duration_ms())
  // does not compile. Presumably, this is due to some detail in its libc++.
  if (source_config.trace_duration_ms() != 0)
    watchdogs_.emplace(
        std::piecewise_construct, std::forward_as_tuple(id),
        std::forward_as_tuple(2 * source_config.trace_duration_ms()));
}

void ProbesProducer::CreateProcessStatsDataSourceInstance(
    const DataSourceConfig& source_config) {
  auto trace_writer = endpoint_->CreateTraceWriter(
      static_cast<BufferID>(source_config.target_buffer()));
  procfs_utils::ProcessMap processes;
  auto trace_packet = trace_writer->NewTracePacket();
  protos::pbzero::ProcessTree* process_tree = trace_packet->set_process_tree();

  file_utils::ForEachPidInProcPath(
      "/proc", [&processes, &process_tree](int pid) {
        if (!processes.count(pid)) {
          if (procfs_utils::ReadTgid(pid) != pid)
            return;
          processes[pid] = procfs_utils::ReadProcessInfo(pid);
        }
        ProcessInfo* process = processes[pid].get();
        procfs_utils::ReadProcessThreads(process);
        auto* process_writer = process_tree->add_processes();
        process_writer->set_pid(process->pid);
        process_writer->set_ppid(process->ppid);
        for (const auto& field : process->cmdline)
          process_writer->add_cmdline(field.c_str());
        for (auto& thread : process->threads) {
          auto* thread_writer = process_writer->add_threads();
          thread_writer->set_tid(thread.second.tid);
          thread_writer->set_name(thread.second.name);
        }
      });
  trace_packet->Finalize();
}

void ProbesProducer::TearDownDataSourceInstance(DataSourceInstanceID id) {
  PERFETTO_LOG("Producer stop (id=%" PRIu64 ")", id);
  PERFETTO_DCHECK(instances_.count(id));
  if (instances_[id] == kFtraceSourceName) {
    size_t removed = delegates_.erase(id);
    PERFETTO_DCHECK(removed == 1);
    // Might return 0 if trace_duration_ms == 0.
    watchdogs_.erase(id);
  }
}

void ProbesProducer::ConnectWithRetries(const char* socket_name,
                                        base::TaskRunner* task_runner) {
  PERFETTO_DCHECK(state_ == kNotStarted);
  state_ = kNotConnected;

  ResetConnectionBackoff();
  socket_name_ = socket_name;
  task_runner_ = task_runner;
  Connect();
}

void ProbesProducer::Connect() {
  PERFETTO_DCHECK(state_ == kNotConnected);
  state_ = kConnecting;
  endpoint_ = ProducerIPCClient::Connect(socket_name_, this, task_runner_);
}

void ProbesProducer::IncreaseConnectionBackoff() {
  connection_backoff_ms_ *= 2;
  if (connection_backoff_ms_ > kMaxConnectionBackoffMs)
    connection_backoff_ms_ = kMaxConnectionBackoffMs;
}

void ProbesProducer::ResetConnectionBackoff() {
  connection_backoff_ms_ = kInitialConnectionBackoffMs;
}

ProbesProducer::SinkDelegate::SinkDelegate(std::unique_ptr<TraceWriter> writer)
    : writer_(std::move(writer)) {}

ProbesProducer::SinkDelegate::~SinkDelegate() = default;

ProbesProducer::FtraceBundleHandle
ProbesProducer::SinkDelegate::GetBundleForCpu(size_t) {
  trace_packet_ = writer_->NewTracePacket();
  return FtraceBundleHandle(trace_packet_->set_ftrace_events());
}

void ProbesProducer::SinkDelegate::OnBundleComplete(size_t,
                                                    FtraceBundleHandle) {
  trace_packet_->Finalize();
}

}  // namespace perfetto
<commit_msg>Add constant to ftrace watchdog. am: 94efaeb36a am: 31874c3397 am: 597b459f8a<commit_after>/*
 * Copyright (C) 2018 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "src/traced/probes/probes_producer.h"

#include <stdio.h>
#include <string>

#include "perfetto/base/logging.h"
#include "perfetto/traced/traced.h"
#include "perfetto/tracing/core/data_source_config.h"
#include "perfetto/tracing/core/data_source_descriptor.h"
#include "perfetto/tracing/core/ftrace_config.h"
#include "perfetto/tracing/core/trace_config.h"
#include "perfetto/tracing/core/trace_packet.h"

#include "src/process_stats/file_utils.h"
#include "src/process_stats/procfs_utils.h"

#include "perfetto/trace/ftrace/ftrace_event_bundle.pbzero.h"
#include "perfetto/trace/ps/process_tree.pbzero.h"
#include "perfetto/trace/trace_packet.pbzero.h"

namespace perfetto {
namespace {

uint64_t kInitialConnectionBackoffMs = 100;
uint64_t kMaxConnectionBackoffMs = 30 * 1000;
const char* kFtraceSourceName = "com.google.perfetto.ftrace";
const char* kProcessStatsSourceName = "com.google.perfetto.process_stats";

}  // namespace.

// State transition diagram:
//                    +----------------------------+
//                    v                            +
// NotStarted -> NotConnected -> Connecting -> Connected
//                    ^              +
//                    +--------------+
//

ProbesProducer::~ProbesProducer() = default;

void ProbesProducer::OnConnect() {
  PERFETTO_DCHECK(state_ == kConnecting);
  state_ = kConnected;
  ResetConnectionBackoff();
  PERFETTO_LOG("Connected to the service");

  DataSourceDescriptor ftrace_descriptor;
  ftrace_descriptor.set_name(kFtraceSourceName);
  endpoint_->RegisterDataSource(ftrace_descriptor,
                                [](DataSourceInstanceID id) {});

  DataSourceDescriptor process_stats_descriptor;
  process_stats_descriptor.set_name(kProcessStatsSourceName);
  endpoint_->RegisterDataSource(process_stats_descriptor,
                                [](DataSourceInstanceID id) {});
}

void ProbesProducer::OnDisconnect() {
  PERFETTO_DCHECK(state_ == kConnected || state_ == kConnecting);
  state_ = kNotConnected;
  PERFETTO_LOG("Disconnected from tracing service");
  IncreaseConnectionBackoff();

  // TODO(hjd): Erase all sinks and add e2e test for this.
  task_runner_->PostDelayedTask([this] { this->Connect(); },
                                connection_backoff_ms_);
}

void ProbesProducer::CreateDataSourceInstance(
    DataSourceInstanceID id,
    const DataSourceConfig& source_config) {
  instances_[id] = source_config.name();
  // PERFETTO_LOG("DataSourceInstanceID: %llu", id);
  if (source_config.name() == kFtraceSourceName) {
    CreateFtraceDataSourceInstance(id, source_config);
  } else if (source_config.name() == kProcessStatsSourceName) {
    CreateProcessStatsDataSourceInstance(source_config);
  } else {
    PERFETTO_ELOG("Data source name: %s not recognised.",
                  source_config.name().c_str());
  }
}

void ProbesProducer::CreateFtraceDataSourceInstance(
    DataSourceInstanceID id,
    const DataSourceConfig& source_config) {
  // Don't retry if FtraceController::Create() failed once.
  // This can legitimately happen on user builds where we cannot access the
  // debug paths, e.g., because of SELinux rules.
  if (ftrace_creation_failed_)
    return;

  // Lazily create on the first instance.
  if (!ftrace_) {
    ftrace_ = FtraceController::Create(task_runner_);

    if (!ftrace_) {
      PERFETTO_ELOG("Failed to create FtraceController");
      ftrace_creation_failed_ = true;
      return;
    }

    ftrace_->DisableAllEvents();
    ftrace_->ClearTrace();
  }

  PERFETTO_LOG("Ftrace start (id=%" PRIu64 ", target_buf=%" PRIu32 ")", id,
               source_config.target_buffer());

  FtraceConfig proto_config = source_config.ftrace_config();

  // TODO(hjd): Static cast is bad, target_buffer() should return a BufferID.
  auto trace_writer = endpoint_->CreateTraceWriter(
      static_cast<BufferID>(source_config.target_buffer()));
  auto delegate =
      std::unique_ptr<SinkDelegate>(new SinkDelegate(std::move(trace_writer)));
  auto sink = ftrace_->CreateSink(std::move(proto_config), delegate.get());
  PERFETTO_CHECK(sink);
  delegate->sink(std::move(sink));
  delegates_.emplace(id, std::move(delegate));
  // Building on Android, watchdogs_.emplace(id, 2* source_config.duration_ms())
  // does not compile. Presumably, this is due to some detail in its libc++.
  // Add constant (5 s, semi arbitrarily) to account for very short trace
  // durations.
  if (source_config.trace_duration_ms() != 0)
    watchdogs_.emplace(
        std::piecewise_construct, std::forward_as_tuple(id),
        std::forward_as_tuple(5000 + 2 * source_config.trace_duration_ms()));
}

void ProbesProducer::CreateProcessStatsDataSourceInstance(
    const DataSourceConfig& source_config) {
  auto trace_writer = endpoint_->CreateTraceWriter(
      static_cast<BufferID>(source_config.target_buffer()));
  procfs_utils::ProcessMap processes;
  auto trace_packet = trace_writer->NewTracePacket();
  protos::pbzero::ProcessTree* process_tree = trace_packet->set_process_tree();

  file_utils::ForEachPidInProcPath(
      "/proc", [&processes, &process_tree](int pid) {
        if (!processes.count(pid)) {
          if (procfs_utils::ReadTgid(pid) != pid)
            return;
          processes[pid] = procfs_utils::ReadProcessInfo(pid);
        }
        ProcessInfo* process = processes[pid].get();
        procfs_utils::ReadProcessThreads(process);
        auto* process_writer = process_tree->add_processes();
        process_writer->set_pid(process->pid);
        process_writer->set_ppid(process->ppid);
        for (const auto& field : process->cmdline)
          process_writer->add_cmdline(field.c_str());
        for (auto& thread : process->threads) {
          auto* thread_writer = process_writer->add_threads();
          thread_writer->set_tid(thread.second.tid);
          thread_writer->set_name(thread.second.name);
        }
      });
  trace_packet->Finalize();
}

void ProbesProducer::TearDownDataSourceInstance(DataSourceInstanceID id) {
  PERFETTO_LOG("Producer stop (id=%" PRIu64 ")", id);
  PERFETTO_DCHECK(instances_.count(id));
  if (instances_[id] == kFtraceSourceName) {
    size_t removed = delegates_.erase(id);
    PERFETTO_DCHECK(removed == 1);
    // Might return 0 if trace_duration_ms == 0.
    watchdogs_.erase(id);
  }
}

void ProbesProducer::ConnectWithRetries(const char* socket_name,
                                        base::TaskRunner* task_runner) {
  PERFETTO_DCHECK(state_ == kNotStarted);
  state_ = kNotConnected;

  ResetConnectionBackoff();
  socket_name_ = socket_name;
  task_runner_ = task_runner;
  Connect();
}

void ProbesProducer::Connect() {
  PERFETTO_DCHECK(state_ == kNotConnected);
  state_ = kConnecting;
  endpoint_ = ProducerIPCClient::Connect(socket_name_, this, task_runner_);
}

void ProbesProducer::IncreaseConnectionBackoff() {
  connection_backoff_ms_ *= 2;
  if (connection_backoff_ms_ > kMaxConnectionBackoffMs)
    connection_backoff_ms_ = kMaxConnectionBackoffMs;
}

void ProbesProducer::ResetConnectionBackoff() {
  connection_backoff_ms_ = kInitialConnectionBackoffMs;
}

ProbesProducer::SinkDelegate::SinkDelegate(std::unique_ptr<TraceWriter> writer)
    : writer_(std::move(writer)) {}

ProbesProducer::SinkDelegate::~SinkDelegate() = default;

ProbesProducer::FtraceBundleHandle
ProbesProducer::SinkDelegate::GetBundleForCpu(size_t) {
  trace_packet_ = writer_->NewTracePacket();
  return FtraceBundleHandle(trace_packet_->set_ftrace_events());
}

void ProbesProducer::SinkDelegate::OnBundleComplete(size_t,
                                                    FtraceBundleHandle) {
  trace_packet_->Finalize();
}

}  // namespace perfetto
<|endoftext|>


<commit_before>/*  cbr
 *  CSFQODPFlowScheduler.cpp
 *
 *  Copyright (c) 2010, Ewen Cheslack-Postava
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are
 *  met:
 *  * Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *  * Neither the name of cbr nor the names of its contributors may
 *    be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "CSFQODPFlowScheduler.hpp"
#include "ServerWeightCalculator.hpp"
#include "LocationService.hpp"
#include "CoordinateSegmentation.hpp"
#include "Random.hpp"
#include "craq_oseg/CraqEntry.hpp"
#define _Ka (Duration::milliseconds((int64)200))
#define _Ka_double (_Ka.toSeconds())

#define KALPHA 29 // Max times fair rate can be decreased during interval

#define CSFQLOG(level, msg) SILOG(csfqodp,level, mContext->id() << "->" << mDestServer << ": " << msg)

namespace CBR {

CSFQODPFlowScheduler::CSFQODPFlowScheduler(SpaceContext* ctx, ForwarderServiceQueue* parent, ServerID sid, uint32 serv_id, uint32 max_size, LocationService* loc)
 : ODPFlowScheduler(ctx, parent, sid, serv_id),
   mQueueBuffer(),
   mQueue(Sirikata::SizedResourceMonitor(max_size)),
   mNeedsNotification(true),
   mLoc(loc),
   mArrivalRate(_Ka_double),
   mAcceptedRate(_Ka_double),
   mCapacityRate(_Ka_double),
   mAlpha(0.0),
   mAlphaWindowed(0.0),
   mCongested(true),
   mCongestionStartTime(Time::null()),
   mCongestionWindow(_Ka),
   mKAlphaReductionsLeft(KALPHA),
   mTotalActiveWeight(0)
{
    mWeightCalculator = WeightCalculatorFactory(mContext->cseg());

    for(int i = 0; i < NUM_DOWNSTREAM; i++)
        mTotalUsedWeight[i] = 0.0;
}

CSFQODPFlowScheduler::~CSFQODPFlowScheduler() {
    delete mWeightCalculator;

#ifdef CSFQODP_DEBUG
    CSFQLOG(warn,"Flow");
    for(FlowMap::iterator flow_it = mFlows.begin(); flow_it != mFlows.end(); flow_it++) {
        ObjectPair op = flow_it->first;
        const FlowInfo& fi = flow_it->second;
        CSFQLOG(warn,"  " <<
            "[" << op.source.toString() << ":" << op.dest.toString() << "] " <<
            "weight: " << fi.weight <<
            " sused: " << fi.usedWeight[SENDER] <<
            " rused: " << fi.usedWeight[RECEIVER] <<
            " -> arrived: " << fi.arrived <<
            " accepted: " << fi.accepted
        );
    }
#endif
}

// ODP push interface
bool CSFQODPFlowScheduler::push(CBR::Protocol::Object::ObjectMessage* msg, const CraqEntry&source_entry, const CraqEntry& dest_entry) {
    boost::lock_guard<boost::mutex> lck(mPushMutex); // FIXME

    ObjectPair op(msg->source_object(), msg->dest_object());
    FlowInfo* flow_info = getFlow(op,source_entry,dest_entry);

    // FIXME update weights, due to possible movement?
    double weight = flow_info->weight;

    // Priority computation failure...
    if (!weight) {
        return false;
    }

    int32 packet_size = msg->ByteSize();
    Time curtime = mContext->recentSimTime();

#ifdef CSFQODP_DEBUG
    flow_info->arrived += packet_size;
#endif

    double label = 0;
#define _edge true // Maybe someday we'll bother with core routers
    if (_edge) {
        // Remove old used weight
        for(int i = 0; i < NUM_DOWNSTREAM; i++)
            mTotalUsedWeight[i] -= flow_info->usedWeight[i];

        // Compute label, updating the rate
        double flow_rate = flow_info->rate.estimate_rate(curtime, packet_size, _Ka_double);
        double w_norm = normalizedFlowWeight(weight);
        label = flow_rate / w_norm;

        // If we were setting label annotations, we would do it here
        // setLabel(p,label);

        // Add updated used weight
        // For acc_rate and total_rates, use_global_values controls
        // whether we're using the values for the entire fair queue
        // (which must be collected downstream and reported back) or
        // the local values (which can obviously only take into
        // account flows we know about).  Note that the correct
        // setting is use_global_values == true.
        double sender_acc_rate = std::max(mSenderCapacity, 1.0);
        // Using the max avoids possible zeros
        double sender_total_weights = std::max(mSenderTotalWeight, weight);
        flow_info->usedWeight[SENDER] = std::min(flow_rate * (sender_total_weights / sender_acc_rate), weight);

        double receiver_acc_rate = std::max(mReceiverCapacity, 1.0);
        // Using the max avoids possible zeros
        double receiver_total_weights = std::max(mReceiverTotalWeight, weight);
        flow_info->usedWeight[RECEIVER] = std::min(flow_rate * (receiver_total_weights / receiver_acc_rate), weight);

        for(int i = 0; i < NUM_DOWNSTREAM; i++)
            mTotalUsedWeight[i] += flow_info->usedWeight[i];
    }

    double prob_drop = std::max(0.0, label ? 1.0 - mAlpha / label : 0);

    // Initialization and reinitialization case: if mAlpha is 0, allow
    // everything through.
    if (mAlpha != 0.0) {
        if ((randFloat() < prob_drop)) {
            estimateAlpha(packet_size, curtime, label, true);
            return false;
        }
    }

    // We're not labelling packets, but in case we do at some point, this is
    // where we'd do it.
    //if (prob_drop > 0) {
        // Annotate packet, label = mAlpha
    //}

    // Try to enqueue.
    Message* serv_msg = createMessageFromODP(msg, mDestServer);
    QueuedMessage qmsg(serv_msg, packet_size);
    bool enqueue_success = mQueue.push(qmsg, false);
    // If we overflowed, drop and adjust alpha
    if (!enqueue_success) {
        if (mKAlphaReductionsLeft-- >= 0)
            mAlpha *= 0.99;
        delete qmsg.msg;
        return false;
    }

    // Finally, restimate alpha.
#ifdef CSFQODP_DEBUG
    flow_info->accepted += packet_size;
#endif
    estimateAlpha(packet_size, curtime, label, false);

    if (mNeedsNotification) {
        mNeedsNotification = false;
        notifyPushFront();
    }

    return true;
}

void CSFQODPFlowScheduler::estimateAlpha(int32 packet_size, Time& arrival_time, double label, bool dropped) {
    mArrivalRate.estimate_rate(arrival_time, packet_size);
    if (!dropped)
        mAcceptedRate.estimate_rate(arrival_time, packet_size);

    // compute the initial value of mAlpha
    if (mAlpha == 0.) {
        if (!queueExceedsLowWaterMark()) {
            if (mAlphaWindowed < label) mAlphaWindowed = label;
            return;
        }
        if (mAlpha < mAlphaWindowed)
            mAlpha = mAlphaWindowed;
        if (mAlpha == 0.)
            mAlpha = minCongestedAlpha(); // arbitrary initialization
        mAlphaWindowed = 0.;
    }

    // Update mAlpha

    // We need to deal with both down stream queues. The capacity dedicated to
    // us is the only parameter dependent on the downstream queues.  Therefore,
    // we compute for both and work with the minimum.
    double sender_cap = mSenderCapacity;
    if (mSenderTotalWeight != 0.0) sender_cap *= mTotalUsedWeight[SENDER] / mSenderTotalWeight;
    double receiver_cap = mReceiverCapacity;
    if (mReceiverTotalWeight != 0.0) receiver_cap *= mTotalUsedWeight[RECEIVER] / mReceiverTotalWeight;

    double cap = std::min(sender_cap, receiver_cap);

    if (cap <= mArrivalRate.get()) { // link congested
        if (!mCongested) { // uncongested -> congested
            mCongested = true;
            mCongestionStartTime = arrival_time;
            mKAlphaReductionsLeft = KALPHA;
        } else {
            if (arrival_time < mCongestionStartTime + mCongestionWindow)
                return;
            mCongestionStartTime = arrival_time;
            mAlpha *= cap/mAcceptedRate.get(arrival_time, _Ka_double);
            if (cap < mAlpha) {
                mAlpha = cap;
            }
        }
    } else {  // (_capacity_rate.get() > mArrivalRate.get()) => link uncongested
        if (mCongested) { // congested -> uncongested
            mCongested = false;
            mCongestionStartTime = arrival_time;
            mAlphaWindowed = 0;
        } else {
            if (arrival_time < mCongestionStartTime + mCongestionWindow) {
                if (mAlphaWindowed < label) mAlphaWindowed = label;
            } else {
                mAlpha = mAlphaWindowed;
                mCongestionStartTime = arrival_time;
                if (!queueExceedsLowWaterMark())
                    mAlpha = 0.;
                else
                    mAlphaWindowed = 0.;
            }
        }
    }
}

static CSFQODPFlowScheduler::Type null_response = NULL;

const CSFQODPFlowScheduler::Type& CSFQODPFlowScheduler::front() const {
    if (mQueueBuffer.msg == NULL) {
        bool avail = mQueue.pop(mQueueBuffer);
        if (!avail) {
            mNeedsNotification = true;
            return null_response;
        }
    }

    return mQueueBuffer.msg;
}

CSFQODPFlowScheduler::Type& CSFQODPFlowScheduler::front() {
    if (mQueueBuffer.msg == NULL) {
        bool avail = mQueue.pop(mQueueBuffer);
        if (!avail) {
            mNeedsNotification = true;
            return null_response;
        }
    }

    return mQueueBuffer.msg;
}

bool CSFQODPFlowScheduler::empty() const {
    bool is_empty = mQueueBuffer.msg == NULL && mQueue.probablyEmpty();
    if (is_empty) mNeedsNotification = true;
    return is_empty;
}

CSFQODPFlowScheduler::Type CSFQODPFlowScheduler::pop() {
    front(); // Prime front element
    QueuedMessage result = mQueueBuffer;
    mQueueBuffer = QueuedMessage();
    if (result.msg == NULL)
        return NULL;

    // If the queue reads as empty after a pop, we're going to need
    // notification.  Otherwise, even if it got emptied, we weren't able to
    // observe it before it got another element.
    front(); // Reprimes front element, might mark for notification on next round

    mCapacityRate.estimate_rate(mContext->recentSimTime(), result.size());
    return result.msg;
}

// Get the sum of the weights of active queues.
float CSFQODPFlowScheduler::totalActiveWeight() {
    return mTotalActiveWeight;
}

// Get the total used weight of active queues.  If all flows are saturating,
// this should equal totalActiveWeights, otherwise it will be smaller.
float CSFQODPFlowScheduler::totalSenderUsedWeight() {
    // No flow tracking, so we just give the entire server weight
    return mTotalUsedWeight[SENDER];
}

// Get the total used weight of active queues.  If all flows are saturating,
// this should equal totalActiveWeights, otherwise it will be smaller.
float CSFQODPFlowScheduler::totalReceiverUsedWeight() {
    return mTotalUsedWeight[RECEIVER];
}

BoundingBox3f CSFQODPFlowScheduler::getObjectWeightRegion(const UUID& objid, const CraqEntry& info) const {
    // We might have exact info
    if (mLoc->contains(objid)) {
        Vector3f pos = mLoc->currentPosition(objid);
        BoundingSphere3f bounds = mLoc->bounds(objid);
        BoundingBox3f bb(pos + bounds.center(), bounds.radius());
        return bb;
    }

    if (info.server() == mContext->id())
        CSFQLOG(warn,"Using approximation for local object!");
    if (info.radius()==1.0) {
        CSFQLOG(warn,"Radius approximation failure! (migration? should we do a cache lookup)");
    }
    // Otherwise, we need to use server info
    // Blech, why is this a bbox list?
    BoundingBoxList server_bbox_list = mContext->cseg()->serverRegion(info.server());
    BoundingBox3f server_bbox = BoundingBox3f::null();
    for(uint32 i = 0; i < server_bbox_list.size(); i++)
        server_bbox.mergeIn(server_bbox_list[i]);
    return BoundingBox3f(server_bbox.center(), info.radius());
}

CSFQODPFlowScheduler::FlowInfo* CSFQODPFlowScheduler::getFlow(const ObjectPair& new_packet_pair, const CraqEntry&source_info, const CraqEntry&dst_info) {
    FlowMap::iterator where = mFlows.find(new_packet_pair);
    if (where==mFlows.end()) {
        BoundingBox3f source_bbox = getObjectWeightRegion(new_packet_pair.source, source_info);
        BoundingBox3f dest_bbox = getObjectWeightRegion(new_packet_pair.dest, dst_info);

        double weight = mWeightCalculator->weight(source_bbox, dest_bbox);

        std::pair<FlowMap::iterator, bool> ins_it = mFlows.insert(FlowMap::value_type(new_packet_pair,FlowInfo(weight)));
        assert(ins_it.second == true);

        mTotalActiveWeight += weight;
        for(int i = 0; i < NUM_DOWNSTREAM; i++)
            mTotalUsedWeight[i] += weight;

        where = ins_it.first;
    }
    return &(where->second);
}

void CSFQODPFlowScheduler::removeFlow(const ObjectPair& packet_pair) {
    FlowMap::iterator where = mFlows.find(packet_pair);
    assert(where != mFlows.end());
    assert(0);
    FlowInfo* fi = &(where->second);
    mTotalActiveWeight -= fi->weight;
    for(int i = 0; i < NUM_DOWNSTREAM; i++)
        mTotalUsedWeight[i] -= fi->usedWeight[i];
    fi = NULL;
    mFlows.erase(where);
}

int CSFQODPFlowScheduler::flowCount() const {
    return mFlows.size();
}

float CSFQODPFlowScheduler::normalizedFlowWeight(float unnorm_weight) {
    // We need normalized weights or else things won't add up properly to give
    // us C total output.  The paper ignores this, presumably because they have
    // a static set of weights which they have already normalized.
    return unnorm_weight / mTotalActiveWeight;
}

} // namespace CBR
<commit_msg>senderTotalWeight should never be less than savedUsedWeight rather than weight total<commit_after>/*  cbr
 *  CSFQODPFlowScheduler.cpp
 *
 *  Copyright (c) 2010, Ewen Cheslack-Postava
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are
 *  met:
 *  * Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *  * Neither the name of cbr nor the names of its contributors may
 *    be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "CSFQODPFlowScheduler.hpp"
#include "ServerWeightCalculator.hpp"
#include "LocationService.hpp"
#include "CoordinateSegmentation.hpp"
#include "Random.hpp"
#include "craq_oseg/CraqEntry.hpp"
#define _Ka (Duration::milliseconds((int64)200))
#define _Ka_double (_Ka.toSeconds())

#define KALPHA 29 // Max times fair rate can be decreased during interval

#define CSFQLOG(level, msg) SILOG(csfqodp,level, mContext->id() << "->" << mDestServer << ": " << msg)

namespace CBR {

CSFQODPFlowScheduler::CSFQODPFlowScheduler(SpaceContext* ctx, ForwarderServiceQueue* parent, ServerID sid, uint32 serv_id, uint32 max_size, LocationService* loc)
 : ODPFlowScheduler(ctx, parent, sid, serv_id),
   mQueueBuffer(),
   mQueue(Sirikata::SizedResourceMonitor(max_size)),
   mNeedsNotification(true),
   mLoc(loc),
   mArrivalRate(_Ka_double),
   mAcceptedRate(_Ka_double),
   mCapacityRate(_Ka_double),
   mAlpha(0.0),
   mAlphaWindowed(0.0),
   mCongested(true),
   mCongestionStartTime(Time::null()),
   mCongestionWindow(_Ka),
   mKAlphaReductionsLeft(KALPHA),
   mTotalActiveWeight(0)
{
    mWeightCalculator = WeightCalculatorFactory(mContext->cseg());

    for(int i = 0; i < NUM_DOWNSTREAM; i++)
        mTotalUsedWeight[i] = 0.0;
}

CSFQODPFlowScheduler::~CSFQODPFlowScheduler() {
    delete mWeightCalculator;

#ifdef CSFQODP_DEBUG
    CSFQLOG(warn,"Flow");
    for(FlowMap::iterator flow_it = mFlows.begin(); flow_it != mFlows.end(); flow_it++) {
        ObjectPair op = flow_it->first;
        const FlowInfo& fi = flow_it->second;
        CSFQLOG(warn,"  " <<
            "[" << op.source.toString() << ":" << op.dest.toString() << "] " <<
            "weight: " << fi.weight <<
            " sused: " << fi.usedWeight[SENDER] <<
            " rused: " << fi.usedWeight[RECEIVER] <<
            " -> arrived: " << fi.arrived <<
            " accepted: " << fi.accepted
        );
    }
#endif
}

// ODP push interface
bool CSFQODPFlowScheduler::push(CBR::Protocol::Object::ObjectMessage* msg, const CraqEntry&source_entry, const CraqEntry& dest_entry) {
    boost::lock_guard<boost::mutex> lck(mPushMutex); // FIXME

    ObjectPair op(msg->source_object(), msg->dest_object());
    FlowInfo* flow_info = getFlow(op,source_entry,dest_entry);

    // FIXME update weights, due to possible movement?
    double weight = flow_info->weight;

    // Priority computation failure...
    if (!weight) {
        return false;
    }

    int32 packet_size = msg->ByteSize();
    Time curtime = mContext->recentSimTime();

#ifdef CSFQODP_DEBUG
    flow_info->arrived += packet_size;
#endif

    double label = 0;
#define _edge true // Maybe someday we'll bother with core routers
    if (_edge) {
        // Remove old used weight
        double savedTotalUsedWeight[NUM_DOWNSTREAM]={0};

        for(int i = 0; i < NUM_DOWNSTREAM; i++) {
            savedTotalUsedWeight[i]=mTotalUsedWeight[i];
            mTotalUsedWeight[i] -= flow_info->usedWeight[i];
        }
        // Compute label, updating the rate
        double flow_rate = flow_info->rate.estimate_rate(curtime, packet_size, _Ka_double);
        double w_norm = normalizedFlowWeight(weight);
        label = flow_rate / w_norm;

        // If we were setting label annotations, we would do it here
        // setLabel(p,label);

        // Add updated used weight
        // For acc_rate and total_rates, use_global_values controls
        // whether we're using the values for the entire fair queue
        // (which must be collected downstream and reported back) or
        // the local values (which can obviously only take into
        // account flows we know about).  Note that the correct
        // setting is use_global_values == true.
        double sender_acc_rate = std::max(mSenderCapacity, 1.0);
        // Using the max avoids possible zeros
        double sender_total_weights = std::max(mSenderTotalWeight, savedTotalUsedWeight[SENDER]);
        flow_info->usedWeight[SENDER] = std::min(flow_rate * (sender_total_weights / sender_acc_rate), weight);

        double receiver_acc_rate = std::max(mReceiverCapacity, 1.0);
        // Using the max avoids possible zeros
        double receiver_total_weights = std::max(mReceiverTotalWeight, savedTotalUsedWeight[RECEIVER]);
        flow_info->usedWeight[RECEIVER] = std::min(flow_rate * (receiver_total_weights / receiver_acc_rate), weight);

        for(int i = 0; i < NUM_DOWNSTREAM; i++)
            mTotalUsedWeight[i] += flow_info->usedWeight[i];
    }

    double prob_drop = std::max(0.0, label ? 1.0 - mAlpha / label : 0);

    // Initialization and reinitialization case: if mAlpha is 0, allow
    // everything through.
    if (mAlpha != 0.0) {
        if ((randFloat() < prob_drop)) {
            estimateAlpha(packet_size, curtime, label, true);
            return false;
        }
    }

    // We're not labelling packets, but in case we do at some point, this is
    // where we'd do it.
    //if (prob_drop > 0) {
        // Annotate packet, label = mAlpha
    //}

    // Try to enqueue.
    Message* serv_msg = createMessageFromODP(msg, mDestServer);
    QueuedMessage qmsg(serv_msg, packet_size);
    bool enqueue_success = mQueue.push(qmsg, false);
    // If we overflowed, drop and adjust alpha
    if (!enqueue_success) {
        if (mKAlphaReductionsLeft-- >= 0)
            mAlpha *= 0.99;
        delete qmsg.msg;
        return false;
    }

    // Finally, restimate alpha.
#ifdef CSFQODP_DEBUG
    flow_info->accepted += packet_size;
#endif
    estimateAlpha(packet_size, curtime, label, false);

    if (mNeedsNotification) {
        mNeedsNotification = false;
        notifyPushFront();
    }

    return true;
}

void CSFQODPFlowScheduler::estimateAlpha(int32 packet_size, Time& arrival_time, double label, bool dropped) {
    mArrivalRate.estimate_rate(arrival_time, packet_size);
    if (!dropped)
        mAcceptedRate.estimate_rate(arrival_time, packet_size);

    // compute the initial value of mAlpha
    if (mAlpha == 0.) {
        if (!queueExceedsLowWaterMark()) {
            if (mAlphaWindowed < label) mAlphaWindowed = label;
            return;
        }
        if (mAlpha < mAlphaWindowed)
            mAlpha = mAlphaWindowed;
        if (mAlpha == 0.)
            mAlpha = minCongestedAlpha(); // arbitrary initialization
        mAlphaWindowed = 0.;
    }

    // Update mAlpha

    // We need to deal with both down stream queues. The capacity dedicated to
    // us is the only parameter dependent on the downstream queues.  Therefore,
    // we compute for both and work with the minimum.
    double sender_cap = mSenderCapacity;
    if (mSenderTotalWeight != 0.0) sender_cap *= mTotalUsedWeight[SENDER] / mSenderTotalWeight;
    double receiver_cap = mReceiverCapacity;
    if (mReceiverTotalWeight != 0.0) receiver_cap *= mTotalUsedWeight[RECEIVER] / mReceiverTotalWeight;

    double cap = std::min(sender_cap, receiver_cap);

    if (cap <= mArrivalRate.get()) { // link congested
        if (!mCongested) { // uncongested -> congested
            mCongested = true;
            mCongestionStartTime = arrival_time;
            mKAlphaReductionsLeft = KALPHA;
        } else {
            if (arrival_time < mCongestionStartTime + mCongestionWindow)
                return;
            mCongestionStartTime = arrival_time;
            mAlpha *= cap/mAcceptedRate.get(arrival_time, _Ka_double);
            if (cap < mAlpha) {
                mAlpha = cap;
            }
        }
    } else {  // (_capacity_rate.get() > mArrivalRate.get()) => link uncongested
        if (mCongested) { // congested -> uncongested
            mCongested = false;
            mCongestionStartTime = arrival_time;
            mAlphaWindowed = 0;
        } else {
            if (arrival_time < mCongestionStartTime + mCongestionWindow) {
                if (mAlphaWindowed < label) mAlphaWindowed = label;
            } else {
                mAlpha = mAlphaWindowed;
                mCongestionStartTime = arrival_time;
                if (!queueExceedsLowWaterMark())
                    mAlpha = 0.;
                else
                    mAlphaWindowed = 0.;
            }
        }
    }
}

static CSFQODPFlowScheduler::Type null_response = NULL;

const CSFQODPFlowScheduler::Type& CSFQODPFlowScheduler::front() const {
    if (mQueueBuffer.msg == NULL) {
        bool avail = mQueue.pop(mQueueBuffer);
        if (!avail) {
            mNeedsNotification = true;
            return null_response;
        }
    }

    return mQueueBuffer.msg;
}

CSFQODPFlowScheduler::Type& CSFQODPFlowScheduler::front() {
    if (mQueueBuffer.msg == NULL) {
        bool avail = mQueue.pop(mQueueBuffer);
        if (!avail) {
            mNeedsNotification = true;
            return null_response;
        }
    }

    return mQueueBuffer.msg;
}

bool CSFQODPFlowScheduler::empty() const {
    bool is_empty = mQueueBuffer.msg == NULL && mQueue.probablyEmpty();
    if (is_empty) mNeedsNotification = true;
    return is_empty;
}

CSFQODPFlowScheduler::Type CSFQODPFlowScheduler::pop() {
    front(); // Prime front element
    QueuedMessage result = mQueueBuffer;
    mQueueBuffer = QueuedMessage();
    if (result.msg == NULL)
        return NULL;

    // If the queue reads as empty after a pop, we're going to need
    // notification.  Otherwise, even if it got emptied, we weren't able to
    // observe it before it got another element.
    front(); // Reprimes front element, might mark for notification on next round

    mCapacityRate.estimate_rate(mContext->recentSimTime(), result.size());
    return result.msg;
}

// Get the sum of the weights of active queues.
float CSFQODPFlowScheduler::totalActiveWeight() {
    return mTotalActiveWeight;
}

// Get the total used weight of active queues.  If all flows are saturating,
// this should equal totalActiveWeights, otherwise it will be smaller.
float CSFQODPFlowScheduler::totalSenderUsedWeight() {
    // No flow tracking, so we just give the entire server weight
    return mTotalUsedWeight[SENDER];
}

// Get the total used weight of active queues.  If all flows are saturating,
// this should equal totalActiveWeights, otherwise it will be smaller.
float CSFQODPFlowScheduler::totalReceiverUsedWeight() {
    return mTotalUsedWeight[RECEIVER];
}

BoundingBox3f CSFQODPFlowScheduler::getObjectWeightRegion(const UUID& objid, const CraqEntry& info) const {
    // We might have exact info
    if (mLoc->contains(objid)) {
        Vector3f pos = mLoc->currentPosition(objid);
        BoundingSphere3f bounds = mLoc->bounds(objid);
        BoundingBox3f bb(pos + bounds.center(), bounds.radius());
        return bb;
    }

    if (info.server() == mContext->id())
        CSFQLOG(warn,"Using approximation for local object!");
    if (info.radius()==1.0) {
        CSFQLOG(warn,"Radius approximation failure! (migration? should we do a cache lookup)");
    }
    // Otherwise, we need to use server info
    // Blech, why is this a bbox list?
    BoundingBoxList server_bbox_list = mContext->cseg()->serverRegion(info.server());
    BoundingBox3f server_bbox = BoundingBox3f::null();
    for(uint32 i = 0; i < server_bbox_list.size(); i++)
        server_bbox.mergeIn(server_bbox_list[i]);
    return BoundingBox3f(server_bbox.center(), info.radius());
}

CSFQODPFlowScheduler::FlowInfo* CSFQODPFlowScheduler::getFlow(const ObjectPair& new_packet_pair, const CraqEntry&source_info, const CraqEntry&dst_info) {
    FlowMap::iterator where = mFlows.find(new_packet_pair);
    if (where==mFlows.end()) {
        BoundingBox3f source_bbox = getObjectWeightRegion(new_packet_pair.source, source_info);
        BoundingBox3f dest_bbox = getObjectWeightRegion(new_packet_pair.dest, dst_info);

        double weight = mWeightCalculator->weight(source_bbox, dest_bbox);

        std::pair<FlowMap::iterator, bool> ins_it = mFlows.insert(FlowMap::value_type(new_packet_pair,FlowInfo(weight)));
        assert(ins_it.second == true);

        mTotalActiveWeight += weight;
        for(int i = 0; i < NUM_DOWNSTREAM; i++)
            mTotalUsedWeight[i] += weight;

        where = ins_it.first;
    }
    return &(where->second);
}

void CSFQODPFlowScheduler::removeFlow(const ObjectPair& packet_pair) {
    FlowMap::iterator where = mFlows.find(packet_pair);
    assert(where != mFlows.end());
    assert(0);
    FlowInfo* fi = &(where->second);
    mTotalActiveWeight -= fi->weight;
    for(int i = 0; i < NUM_DOWNSTREAM; i++)
        mTotalUsedWeight[i] -= fi->usedWeight[i];
    fi = NULL;
    mFlows.erase(where);
}

int CSFQODPFlowScheduler::flowCount() const {
    return mFlows.size();
}

float CSFQODPFlowScheduler::normalizedFlowWeight(float unnorm_weight) {
    // We need normalized weights or else things won't add up properly to give
    // us C total output.  The paper ignores this, presumably because they have
    // a static set of weights which they have already normalized.
    return unnorm_weight / mTotalActiveWeight;
}

} // namespace CBR
<|endoftext|>


<commit_before>/*===================================================================

The Medical Imaging Interaction Toolkit (MITK)

Copyright (c) German Cancer Research Center,
Division of Medical and Biological Informatics.
All rights reserved.

This software is distributed WITHOUT ANY WARRANTY; without
even the implied warranty of MERCHANTABILITY or FITNESS FOR
A PARTICULAR PURPOSE.

See LICENSE.txt or http://www.mitk.org for details.

===================================================================*/

#include <mitkTestingMacros.h>
#include <mitkTestFixture.h>
#include "mitkIOUtil.h"
#include <cmath>

#include <mitkGIFCurvatureStatistic.h>

class mitkGIFLocalIntensityTestSuite : public mitk::TestFixture
{
  CPPUNIT_TEST_SUITE(mitkGIFLocalIntensityTestSuite);

  MITK_TEST(ImageDescription_PhantomTest);

  CPPUNIT_TEST_SUITE_END();

private:
	mitk::Image::Pointer m_IBSI_Phantom_Image_Small;
	mitk::Image::Pointer m_IBSI_Phantom_Image_Large;
	mitk::Image::Pointer m_IBSI_Phantom_Mask_Small;
	mitk::Image::Pointer m_IBSI_Phantom_Mask_Large;

public:

  void setUp(void) override
  {
    m_IBSI_Phantom_Image_Small = mitk::IOUtil::LoadImage(GetTestDataFilePath("Radiomics/IBSI_Phantom_Image_Small.nrrd"));
    m_IBSI_Phantom_Image_Large = mitk::IOUtil::LoadImage(GetTestDataFilePath("Radiomics/IBSI_Phantom_Image_Large.nrrd"));
    m_IBSI_Phantom_Mask_Small = mitk::IOUtil::LoadImage(GetTestDataFilePath("Radiomics/IBSI_Phantom_Mask_Small.nrrd"));
    m_IBSI_Phantom_Mask_Large = mitk::IOUtil::LoadImage(GetTestDataFilePath("Radiomics/IBSI_Phantom_Mask_Large.nrrd"));
  }

  void ImageDescription_PhantomTest()
  {
    mitk::GIFCurvatureStatistic::Pointer featureCalculator = mitk::GIFCurvatureStatistic::New();

    featureCalculator->SetUseBinsize(true);
    featureCalculator->SetBinsize(1.0);
    featureCalculator->SetUseMinimumIntensity(true);
    featureCalculator->SetUseMaximumIntensity(true);
    featureCalculator->SetMinimumIntensity(0.5);
    featureCalculator->SetMaximumIntensity(6.5);

    auto featureList = featureCalculator->CalculateFeatures(m_IBSI_Phantom_Image_Large, m_IBSI_Phantom_Mask_Large);

    std::map<std::string, double> results;
    for (auto valuePair : featureList)
    {
      MITK_INFO << valuePair.first << " : " << valuePair.second;
      results[valuePair.first] = valuePair.second;
    }
    CPPUNIT_ASSERT_EQUAL_MESSAGE("Image Diagnostics should calculate 44 features.", std::size_t(44), featureList.size());

    // These values are obtained by a run of the filter.
    // The might be wrong!
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Minimum Mean Curvature with Large IBSI Phantom Image", -1.51, results["Curvature Feature::Minimum Mean Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Maximum Mean Curvature with Large IBSI Phantom Image", 0.51, results["Curvature Feature::Maximum Mean Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Mean Mean Curvature with Large IBSI Phantom Image", 0.095, results["Curvature Feature::Mean Mean Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Standard Deviation Mean Curvature with Large IBSI Phantom Image", 0.45, results["Curvature Feature::Standard Deviation Mean Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Skewness Mean Curvature with Large IBSI Phantom Image", -2.55, results["Curvature Feature::Skewness Mean Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Mean Positive Mean Curvature with Large IBSI Phantom Image", 0.30, results["Curvature Feature::Mean Positive Mean Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Standard Deviation Positive Mean Curvature with Large IBSI Phantom Image", 0.12, results["Curvature Feature::Standard Deviation Positive Mean Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Skewness Positive Mean Curvature with Large IBSI Phantom Image", 0.60, results["Curvature Feature::Skewness Positive Mean Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Mean Negative Mean Curvature with Large IBSI Phantom Image", -0.955, results["Curvature Feature::Mean Negative Mean Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Standard Deviation Negative Mean Curvature with Large IBSI Phantom Image", 0.56, results["Curvature Feature::Standard Deviation Negative Mean Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Skewness Negative Mean Curvature with Large IBSI Phantom Image", 0.09, results["Curvature Feature::Skewness Negative Mean Curvature"], 0.01);

    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Minimum Gaussian Curvature with Large IBSI Phantom Image", -0.211, results["Curvature Feature::Minimum Gaussian Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Maximum Gaussian Curvature with Large IBSI Phantom Image", 1.81, results["Curvature Feature::Maximum Gaussian Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Mean Gaussian Curvature with Large IBSI Phantom Image", 0.14, results["Curvature Feature::Mean Gaussian Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Standard Deviation Gaussian Curvature with Large IBSI Phantom Image", 0.42, results["Curvature Feature::Standard Deviation Gaussian Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Skewness Gaussian Curvature with Large IBSI Phantom Image", 3.51, results["Curvature Feature::Skewness Gaussian Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Mean Positive Gaussian Curvature with Large IBSI Phantom Image", 0.26, results["Curvature Feature::Mean Positive Gaussian Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Standard Deviation Positive Gaussian Curvature with Large IBSI Phantom Image", 0.53, results["Curvature Feature::Standard Deviation Positive Gaussian Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Skewness Positive Gaussian Curvature with Large IBSI Phantom Image", 2.52, results["Curvature Feature::Skewness Positive Gaussian Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Mean Negative Gaussian Curvature with Large IBSI Phantom Image", -0.03, results["Curvature Feature::Mean Negative Gaussian Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Standard Deviation Negative Gaussian Curvature with Large IBSI Phantom Image", 0.055, results["Curvature Feature::Standard Deviation Negative Gaussian Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Skewness Negative Gaussian Curvature with Large IBSI Phantom Image", -1.92, results["Curvature Feature::Skewness Negative Gaussian Curvature"], 0.01);

    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Minimum Minimum Curvature with Large IBSI Phantom Image", -2.19, results["Curvature Feature::Minimum Minimum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Maximum Minimum Curvature with Large IBSI Phantom Image", 0.35, results["Curvature Feature::Maximum Minimum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Mean Minimum Curvature with Large IBSI Phantom Image", -0.11, results["Curvature Feature::Mean Minimum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Standard Deviation Minimum Curvature with Large IBSI Phantom Image", 0.573, results["Curvature Feature::Standard Deviation Minimum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Skewness Minimum Curvature with Large IBSI Phantom Image", -2.742, results["Curvature Feature::Skewness Minimum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Mean Positive Minimum Curvature with Large IBSI Phantom Image", 0.161, results["Curvature Feature::Mean Positive Minimum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Standard Deviation Positive Minimum Curvature with Large IBSI Phantom Image", 0.165, results["Curvature Feature::Standard Deviation Positive Minimum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Skewness Positive Minimum Curvature with Large IBSI Phantom Image", 0.108, results["Curvature Feature::Skewness Positive Minimum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Mean Negative Minimum Curvature with Large IBSI Phantom Image", -0.42, results["Curvature Feature::Mean Negative Minimum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Standard Deviation Negative Minimum Curvature with Large IBSI Phantom Image", 0.733, results["Curvature Feature::Standard Deviation Negative Minimum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Skewness Negative Minimum Curvature with Large IBSI Phantom Image", -1.73, results["Curvature Feature::Skewness Negative Minimum Curvature"], 0.01);

    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Minimum Maximum Curvature with Large IBSI Phantom Image", -0.83, results["Curvature Feature::Minimum Maximum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Maximum Maximum Curvature with Large IBSI Phantom Image", 0.79, results["Curvature Feature::Maximum Maximum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Mean Maximum Curvature with Large IBSI Phantom Image", 0.30, results["Curvature Feature::Mean Maximum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Standard Deviation Maximum Curvature with Large IBSI Phantom Image", 0.369, results["Curvature Feature::Standard Deviation Maximum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Skewness Maximum Curvature with Large IBSI Phantom Image", -1.617, results["Curvature Feature::Skewness Maximum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Mean Positive Maximum Curvature with Large IBSI Phantom Image", 0.419, results["Curvature Feature::Mean Positive Maximum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Standard Deviation Positive Maximum Curvature with Large IBSI Phantom Image", 0.217, results["Curvature Feature::Standard Deviation Positive Maximum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Skewness Positive Maximum Curvature with Large IBSI Phantom Image", -0.958, results["Curvature Feature::Skewness Positive Maximum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Mean Negative Maximum Curvature with Large IBSI Phantom Image", -0.44, results["Curvature Feature::Mean Negative Maximum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Standard Deviation Negative Maximum Curvature with Large IBSI Phantom Image", 0.399, results["Curvature Feature::Standard Deviation Negative Maximum Curvature"], 0.01);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Curvature Feature::Skewness Negative Maximum Curvature with Large IBSI Phantom Image", 0.109, results["Curvature Feature::Skewness Negative Maximum Curvature"], 0.01);
  }

};

MITK_TEST_SUITE_REGISTRATION(mitkGIFLocalIntensity )<commit_msg>Full structure, wrong output<commit_after>/*===================================================================

The Medical Imaging Interaction Toolkit (MITK)

Copyright (c) German Cancer Research Center,
Division of Medical and Biological Informatics.
All rights reserved.

This software is distributed WITHOUT ANY WARRANTY; without
even the implied warranty of MERCHANTABILITY or FITNESS FOR
A PARTICULAR PURPOSE.

See LICENSE.txt or http://www.mitk.org for details.

===================================================================*/

#include <mitkTestingMacros.h>
#include <mitkTestFixture.h>
#include "mitkIOUtil.h"
#include <cmath>

#include <mitkGIFLocalIntensity.h>

class mitkGIFLocalIntensityTestSuite : public mitk::TestFixture
{
  CPPUNIT_TEST_SUITE(mitkGIFLocalIntensityTestSuite);

  MITK_TEST(ImageDescription_PhantomTest);

  CPPUNIT_TEST_SUITE_END();

private:
	mitk::Image::Pointer m_IBSI_Phantom_Image_Small;
	mitk::Image::Pointer m_IBSI_Phantom_Image_Large;
	mitk::Image::Pointer m_IBSI_Phantom_Mask_Small;
	mitk::Image::Pointer m_IBSI_Phantom_Mask_Large;

public:

  void setUp(void) override
  {
    m_IBSI_Phantom_Image_Small = mitk::IOUtil::LoadImage(GetTestDataFilePath("Radiomics/IBSI_Phantom_Image_Small.nrrd"));
    m_IBSI_Phantom_Image_Large = mitk::IOUtil::LoadImage(GetTestDataFilePath("Radiomics/IBSI_Phantom_Image_Large.nrrd"));
    m_IBSI_Phantom_Mask_Small = mitk::IOUtil::LoadImage(GetTestDataFilePath("Radiomics/IBSI_Phantom_Mask_Small.nrrd"));
    m_IBSI_Phantom_Mask_Large = mitk::IOUtil::LoadImage(GetTestDataFilePath("Radiomics/IBSI_Phantom_Mask_Large.nrrd"));
  }

  void ImageDescription_PhantomTest()
  {
    mitk::GIFLocalIntensity::Pointer featureCalculator = mitk::GIFLocalIntensity::New();

    featureCalculator->SetUseBinsize(true);
    featureCalculator->SetBinsize(3);
    featureCalculator->SetUseMinimumIntensity(true);
    featureCalculator->SetUseMaximumIntensity(true);
    featureCalculator->SetMinimumIntensity(1);
    featureCalculator->SetMaximumIntensity(6);

    auto featureList = featureCalculator->CalculateFeatures(m_IBSI_Phantom_Image_Large, m_IBSI_Phantom_Mask_Large);

    std::map<std::string, double> results;
    for (auto valuePair : featureList)
    {
      MITK_INFO << valuePair.first << " : " << valuePair.second;
      results[valuePair.first] = valuePair.second;
    }
    CPPUNIT_ASSERT_EQUAL_MESSAGE("Image Diagnostics should calculate 44 features.", std::size_t(2), featureList.size());

    // These values are obtained by a run of the filter.
    // The might be wrong!
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Local Intensity::Local Intensity Peak with Large IBSI Phantom Image", 2.6, results["Local Intensity::Local Intensity Peak"], 0.1);
    CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Local Intensity::Global Intensity Peak with Large IBSI Phantom Image", 3.1, results["Local Intensity::Global Intensity Peak"], 0.1);
  }

};

MITK_TEST_SUITE_REGISTRATION(mitkGIFLocalIntensity )<|endoftext|>


<commit_before>/*=========================================================================

Program:   Medical Imaging & Interaction Toolkit
Language:  C++
Date:      $Date$
Version:   $Revision: 1.12 $

Copyright (c) German Cancer Research Center, Division of Medical and
Biological Informatics. All rights reserved.
See MITKCopyright.txt or http://www.mitk.org/copyright.html for details.

This software is distributed WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

#include "mitkImageSerializer.h"

#include "mitkImageWriter.h"
#include <Poco/Path.h>

MITK_REGISTER_SERIALIZER(ImageSerializer)

mitk::ImageSerializer::ImageSerializer()
{
}

mitk::ImageSerializer::~ImageSerializer()
{
}

std::string mitk::ImageSerializer::Serialize()
{
  const Image* image = dynamic_cast<const Image*>( m_Data.GetPointer() );
  if (!image)
  {
    MITK_ERROR << " Object at " << (const void*) this->m_Data
              << " is not an mitk::Image. Cannot serialize as image.";
    return "";
  }

  std::string filename( this->GetUniqueFilenameInWorkingDirectory() );
std::cout << "creating file " << filename << " in " << m_WorkingDirectory << std::endl;
  filename += "_";
  filename += m_FilenameHint;

  std::string fullname(m_WorkingDirectory);
  fullname += Poco::Path::separator();
  fullname += filename;

  try
  {
    ImageWriter::Pointer writer = ImageWriter::New();
    writer->SetFileName( fullname );
    writer->SetExtension(".pic");
    writer->SetInput( const_cast<Image*>(image) ); // bad writer design??
    writer->Write();
    fullname = writer->GetFileName();
  }
  catch (std::exception& e)
  {
    MITK_ERROR << " Error serializing object at " << (const void*) this->m_Data
              << " to "
              << fullname
              << ": "
              << e.what();
    return "";
  }
  return Poco::Path(fullname).getFileName();// + ".pic";
}
<commit_msg>COMP: changes in ImageWriter caused the SceneIOTest to fail. This is due to the appending of the file extension to the filename. This issue was fixed in the ImageSerialzer class.<commit_after>/*=========================================================================

Program:   Medical Imaging & Interaction Toolkit
Language:  C++
Date:      $Date$
Version:   $Revision: 1.12 $

Copyright (c) German Cancer Research Center, Division of Medical and
Biological Informatics. All rights reserved.
See MITKCopyright.txt or http://www.mitk.org/copyright.html for details.

This software is distributed WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

#include "mitkImageSerializer.h"

#include "mitkImageWriter.h"
#include <Poco/Path.h>

MITK_REGISTER_SERIALIZER(ImageSerializer)

mitk::ImageSerializer::ImageSerializer()
{
}

mitk::ImageSerializer::~ImageSerializer()
{
}

std::string mitk::ImageSerializer::Serialize()
{
  const Image* image = dynamic_cast<const Image*>( m_Data.GetPointer() );
  if (!image)
  {
    MITK_ERROR << " Object at " << (const void*) this->m_Data
              << " is not an mitk::Image. Cannot serialize as image.";
    return "";
  }

  std::string filename( this->GetUniqueFilenameInWorkingDirectory() );
std::cout << "creating file " << filename << " in " << m_WorkingDirectory << std::endl;
  filename += "_";
  filename += m_FilenameHint;

  std::string fullname(m_WorkingDirectory);
  fullname += Poco::Path::separator();
  fullname += filename;

  try
  {
    ImageWriter::Pointer writer = ImageWriter::New();
    writer->SetFileName( fullname );
    writer->SetExtension(".pic");
    writer->SetInput( const_cast<Image*>(image) ); // bad writer design??
    writer->Write();
    fullname = writer->GetFileName();
    fullname.append(writer->GetFileExtension());
  }
  catch (std::exception& e)
  {
    MITK_ERROR << " Error serializing object at " << (const void*) this->m_Data
              << " to "
              << fullname
              << ": "
              << e.what();
    return "";
  }
  return Poco::Path(fullname).getFileName();// + ".pic";
}
<|endoftext|>


<commit_before>#include <iostream>
#include <fstream>
#include <ctime>
#include <cstdio>
#include <getopt.h>
#include <sys/stat.h>
#include "gcsa.h"
#include "subcommand.hpp"
// From gcsa2
#include "files.h"
#include "json2pb.h"
#include "vg.hpp"
#include "vg.pb.h"
#include "index.hpp"
#include "mapper.hpp"
#include "Variant.h"
#include "Fasta.h"
#include "stream.hpp"
#include "alignment.hpp"
#include "convert.hpp"
#include "pileup.hpp"
#include "filter.hpp"
#include "google/protobuf/stubs/common.h"
#include "progress_bar.hpp"
#include "vg_git_version.hpp"
#include "IntervalTree.h"
#include "genotyper.hpp"
#include "bubbles.hpp"
#include "translator.hpp"
#include "homogenizer.hpp"
#include "subcommand.hpp"

using namespace std;
using namespace vg;
using namespace vg::subcommand;

void help_homogenize(char** argv){
    cerr << "Usage: " << argv[0] << " homogenize [options] <graph.vg>" << endl
        << "Options:" << endl
        << " -x, --xg   an xg index for generating mappings." << endl
        << " -g, --gcsa a gcsa2 index for generating mappings" << endl
        << " -k, --kmer a kmer size to use for in-memory GCSA/XG construction, if no indicies are provided." << endl
        << endl;
}

int main_homogenize(int argc, char** argv){
    string gcsa_name = "";
    string lcp_name = "";
    string xg_name = "";

    int kmer_size = 11;

    bool debug = false;
    bool in_mem_path_only = false;
    bool build_in_memory = false;

    if (argc <= 2) {
        help_homogenize(argv);
        return 1;
    }

    int c;
    optind = 2; // force optind past command positional argument
    while (true) {
        static struct option long_options[] =
        {
            {"xg", required_argument, 0, 'x'},
            {"gcsa", required_argument, 0, 'g'},
            {"kmer", required_argument, 0, 'k'},
            {0, 0, 0, 0}

        };
        int option_index = 0;
        c = getopt_long (argc, argv, "g:x:k:",
                long_options, &option_index);

        // Detect the end of the options.
        if (c == -1)
            break;

        switch (c)
        {
            case 'x':
                xg_name = optarg;
                break;
            case 'g':
                gcsa_name = optarg;
                lcp_name = gcsa_name + ".lcp";
                break;
            case 'k':
                kmer_size = atoi(optarg);
                break;
            case 'h':
            case '?':
            default:
                help_homogenize(argv);
                abort();
        }

    }

    VG* graph;
    string file_name = argv[optind];
    if (file_name == "-") {
        graph = new VG(std::cin);
    } else {
        ifstream in;
        in.open(file_name.c_str());
        graph = new VG(in);
    }

    gcsa::GCSA* gcsa_index = nullptr;
    gcsa::LCPArray* gcsa_lcp = nullptr;
    xg::XG* xg_index = nullptr;

    // TODO filter by whether a read is on the ref path
    // i.e.:
    // 1. Copy the graph
    VG* o_graph = new VG(*graph);
    cerr << "GRAPH COPIED" << endl;
    // 2. Cache the reference path(s)
    // (Not sure how to grab these, so for now just grab the first path in the graph)
    set<string> kept_paths;
    set<string> removed_paths;
    int i = 0;
    for (auto x : o_graph->paths._paths){
        if (i == 0){
            kept_paths.insert(x.first);
        }
        else{
            removed_paths.insert(x.first);
        }
        i++;
    }
    //
    Paths p = graph->paths;
    //
    // 3. Remove all paths in the graph, except the reference
    (o_graph->paths).remove_paths(removed_paths);
    for ( auto x : (o_graph->paths)._paths){
        cerr << x.first << endl;
    }
    //
    delete graph;

    if (xg_name.empty() && gcsa_name.empty()){
        build_in_memory = true;
    }

    if (build_in_memory) {
        xg_index = new xg::XG(o_graph->graph);
        assert(kmer_size);
        int doubling_steps = 2;
        o_graph->build_gcsa_lcp(gcsa_index, gcsa_lcp, kmer_size, in_mem_path_only, false, 2);
        //delete graph;
    } else {
        // We try opening the file, and then see if it worked
        ifstream xg_stream(xg_name);

        if(xg_stream) {
            // We have an xg index!
            if(debug) {
                cerr << "Loading xg index " << xg_name << "..." << endl;
            }
            xg_index = new xg::XG(xg_stream);
        }

        ifstream gcsa_stream(gcsa_name);
        if(gcsa_stream) {
            // We have a GCSA index too!
            if(debug) {
                cerr << "Loading GCSA2 index " << gcsa_name << "..." << endl;
            }
            gcsa_index = new gcsa::GCSA();
            gcsa_index->load(gcsa_stream);
        }

        string lcp_name = gcsa_name + ".lcp";
        ifstream lcp_stream(lcp_name);
        if (lcp_stream) {
            if(debug) {
                cerr << "Loading LCP index " << gcsa_name << "..." << endl;
            }
            gcsa_lcp = new gcsa::LCPArray();
            gcsa_lcp->load(lcp_stream);
        }
    }

    Homogenizer hh;

    hh.homogenize(o_graph, xg_index, gcsa_index, gcsa_lcp, p, kmer_size);

    /* stream out graph */

    o_graph->serialize_to_ostream(std::cout);
    delete o_graph;

}

static Subcommand vg_homogenize("homogenize", "homogenize augmented graphs", main_homogenize);

<commit_msg>Remove unused and un-Makefiled include<commit_after>#include <iostream>
#include <fstream>
#include <ctime>
#include <cstdio>
#include <getopt.h>
#include <sys/stat.h>
#include "gcsa.h"
#include "subcommand.hpp"
// From gcsa2
#include "files.h"
#include "json2pb.h"
#include "vg.hpp"
#include "vg.pb.h"
#include "index.hpp"
#include "mapper.hpp"
#include "Variant.h"
#include "Fasta.h"
#include "stream.hpp"
#include "alignment.hpp"
#include "convert.hpp"
#include "pileup.hpp"
#include "filter.hpp"
#include "google/protobuf/stubs/common.h"
#include "progress_bar.hpp"
#include "IntervalTree.h"
#include "genotyper.hpp"
#include "bubbles.hpp"
#include "translator.hpp"
#include "homogenizer.hpp"
#include "subcommand.hpp"

using namespace std;
using namespace vg;
using namespace vg::subcommand;

void help_homogenize(char** argv){
    cerr << "Usage: " << argv[0] << " homogenize [options] <graph.vg>" << endl
        << "Options:" << endl
        << " -x, --xg   an xg index for generating mappings." << endl
        << " -g, --gcsa a gcsa2 index for generating mappings" << endl
        << " -k, --kmer a kmer size to use for in-memory GCSA/XG construction, if no indicies are provided." << endl
        << endl;
}

int main_homogenize(int argc, char** argv){
    string gcsa_name = "";
    string lcp_name = "";
    string xg_name = "";

    int kmer_size = 11;

    bool debug = false;
    bool in_mem_path_only = false;
    bool build_in_memory = false;

    if (argc <= 2) {
        help_homogenize(argv);
        return 1;
    }

    int c;
    optind = 2; // force optind past command positional argument
    while (true) {
        static struct option long_options[] =
        {
            {"xg", required_argument, 0, 'x'},
            {"gcsa", required_argument, 0, 'g'},
            {"kmer", required_argument, 0, 'k'},
            {0, 0, 0, 0}

        };
        int option_index = 0;
        c = getopt_long (argc, argv, "g:x:k:",
                long_options, &option_index);

        // Detect the end of the options.
        if (c == -1)
            break;

        switch (c)
        {
            case 'x':
                xg_name = optarg;
                break;
            case 'g':
                gcsa_name = optarg;
                lcp_name = gcsa_name + ".lcp";
                break;
            case 'k':
                kmer_size = atoi(optarg);
                break;
            case 'h':
            case '?':
            default:
                help_homogenize(argv);
                abort();
        }

    }

    VG* graph;
    string file_name = argv[optind];
    if (file_name == "-") {
        graph = new VG(std::cin);
    } else {
        ifstream in;
        in.open(file_name.c_str());
        graph = new VG(in);
    }

    gcsa::GCSA* gcsa_index = nullptr;
    gcsa::LCPArray* gcsa_lcp = nullptr;
    xg::XG* xg_index = nullptr;

    // TODO filter by whether a read is on the ref path
    // i.e.:
    // 1. Copy the graph
    VG* o_graph = new VG(*graph);
    cerr << "GRAPH COPIED" << endl;
    // 2. Cache the reference path(s)
    // (Not sure how to grab these, so for now just grab the first path in the graph)
    set<string> kept_paths;
    set<string> removed_paths;
    int i = 0;
    for (auto x : o_graph->paths._paths){
        if (i == 0){
            kept_paths.insert(x.first);
        }
        else{
            removed_paths.insert(x.first);
        }
        i++;
    }
    //
    Paths p = graph->paths;
    //
    // 3. Remove all paths in the graph, except the reference
    (o_graph->paths).remove_paths(removed_paths);
    for ( auto x : (o_graph->paths)._paths){
        cerr << x.first << endl;
    }
    //
    delete graph;

    if (xg_name.empty() && gcsa_name.empty()){
        build_in_memory = true;
    }

    if (build_in_memory) {
        xg_index = new xg::XG(o_graph->graph);
        assert(kmer_size);
        int doubling_steps = 2;
        o_graph->build_gcsa_lcp(gcsa_index, gcsa_lcp, kmer_size, in_mem_path_only, false, 2);
        //delete graph;
    } else {
        // We try opening the file, and then see if it worked
        ifstream xg_stream(xg_name);

        if(xg_stream) {
            // We have an xg index!
            if(debug) {
                cerr << "Loading xg index " << xg_name << "..." << endl;
            }
            xg_index = new xg::XG(xg_stream);
        }

        ifstream gcsa_stream(gcsa_name);
        if(gcsa_stream) {
            // We have a GCSA index too!
            if(debug) {
                cerr << "Loading GCSA2 index " << gcsa_name << "..." << endl;
            }
            gcsa_index = new gcsa::GCSA();
            gcsa_index->load(gcsa_stream);
        }

        string lcp_name = gcsa_name + ".lcp";
        ifstream lcp_stream(lcp_name);
        if (lcp_stream) {
            if(debug) {
                cerr << "Loading LCP index " << gcsa_name << "..." << endl;
            }
            gcsa_lcp = new gcsa::LCPArray();
            gcsa_lcp->load(lcp_stream);
        }
    }

    Homogenizer hh;

    hh.homogenize(o_graph, xg_index, gcsa_index, gcsa_lcp, p, kmer_size);

    /* stream out graph */

    o_graph->serialize_to_ostream(std::cout);
    delete o_graph;

}

static Subcommand vg_homogenize("homogenize", "homogenize augmented graphs", main_homogenize);

<|endoftext|>