<commit_before>/*
 *  Copyright 2008-2013 NVIDIA Corporation
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

#pragma once

#include <agency/detail/config.hpp>
#include <agency/cuda/detail/feature_test.hpp>
#include <agency/cuda/detail/terminate.hpp>
#include <agency/cuda/detail/unique_ptr.hpp>
#include <agency/cuda/detail/then_kernel.hpp>
#include <agency/cuda/detail/launch_kernel.hpp>
#include <agency/cuda/detail/workaround_unused_variable_warning.hpp>
#include <utility>

namespace agency
{
namespace cuda
{

template<typename T> class future;

template<>
class future<void>
{
  public:
    // XXX this should be private
    __host__ __device__
    future(cudaStream_t s) : future(s, 0) {}

    // XXX stream_ should default to per-thread default stream
    __host__ __device__
    future() : future(0) {}

    __host__ __device__
    future(future&& other)
      : future()
    {
      future::swap(stream_, other.stream_);
      future::swap(event_,  other.event_);
    } // end future()

    __host__ __device__
    future &operator=(future&& other)
    {
      future::swap(stream_, other.stream_);
      future::swap(event_,  other.event_);
      return *this;
    } // end operator=()

    __host__ __device__
    ~future()
    {
      if(valid())
      {
#if __cuda_lib_has_cudart
        // swallow errors
        cudaError_t e = cudaEventDestroy(event_);

#if __cuda_lib_has_printf
        if(e)
        {
          printf("CUDA error after cudaEventDestroy in agency::cuda::future<void> dtor: %s", cudaGetErrorString(e));
        } // end if
#endif // __cuda_lib_has_printf
#endif // __cuda_lib_has_cudart
      } // end if
    } // end ~future()

    __host__ __device__
    void wait() const
    {
      // XXX should probably check for valid() here

#if __cuda_lib_has_cudart

#ifndef __CUDA_ARCH__
      // XXX need to capture the error as an exception and then throw it in .get()
      detail::throw_on_error(cudaEventSynchronize(event_), "cudaEventSynchronize in agency::cuda<void>::future::wait");
#else
      // XXX need to capture the error as an exception and then throw it in .get()
      detail::throw_on_error(cudaDeviceSynchronize(), "cudaDeviceSynchronize in agency::cuda<void>::future::wait");
#endif // __CUDA_ARCH__

#else
      detail::terminate_with_message("agency::cuda::future<void>::wait() requires CUDART");
#endif // __cuda_lib_has_cudart
    } // end wait()

    __host__ __device__
    void get() const
    {
      wait();
    } // end get()

    // XXX we can eliminate this I think
    __host__ __device__
    future<void> discard_value()
    {
      return std::move(*this);
    } // end discard_value()

    __host__ __device__
    bool valid() const
    {
      return event_ != 0;
    } // end valid()

    __host__ __device__
    cudaEvent_t event() const
    {
      return event_;
    } // end event()

    __host__ __device__
    cudaStream_t stream() const
    {
      return stream_;
    } // end stream()

    __host__ __device__
    static future<void> make_ready()
    {
      cudaEvent_t ready_event = 0;

#if __cuda_lib_has_cudart
      detail::throw_on_error(cudaEventCreateWithFlags(&ready_event, event_create_flags), "cudaEventCreateWithFlags in agency::cuda::future<void>::make_ready");
#else
      detail::terminate_with_message("agency::cuda::future<void>::make_ready() requires CUDART");
#endif

      future<void> result;
      result.set_valid(ready_event);

      return result;
    }

    // XXX this is only used by grid_executor::then_execute()
    __host__ __device__
    std::nullptr_t ptr()
    {
      return nullptr;
    }

    template<class Function>
    __host__ __device__
    future<typename std::result_of<Function()>::type>
      then(Function f)
    {
      void (*kernel_ptr)(detail::my_nullptr_t, Function) = detail::then_kernel<Function>;
      detail::workaround_unused_variable_warning(kernel_ptr);

      using result_type = typename std::result_of<Function()>::type;

      using result_future_type = future<result_type>;

      result_future_type result(stream());

      cudaEvent_t next_event = detail::checked_launch_kernel_after_event_returning_next_event(reinterpret_cast<void*>(kernel_ptr), dim3{1}, dim3{1}, 0, stream(), event(), ptr(), f);

      // give next_event to the result future
      result.set_valid(result);

      return result;
    }

    // XXX set_valid() should only be available to friends
    //     such as future<T> and grid_executor
    __host__ __device__
    void set_valid(cudaEvent_t e)
    {
      event_ = e;
    }

  private:
    __host__ __device__
    future(cudaStream_t s, cudaEvent_t e) : stream_(s), event_(e) {}

    // implement swap to avoid depending on thrust::swap
    template<class T>
    __host__ __device__
    static void swap(T& a, T& b)
    {
      T tmp{a};
      a = b;
      b = tmp;
    }

    static const int event_create_flags = cudaEventDisableTiming;

    cudaStream_t stream_;
    cudaEvent_t event_;
}; // end future<void>

template<class T>
class future
{
  public:
    __host__ __device__
    future()
      : completion_()
    {}

    // XXX this should be private
    // XXX this constructor should not even exist
    //     the ready event should be created in completion_'s initializer
    template<class U>
    __host__ __device__
    future(U&& value, future<void>& e)
      : completion_(std::move(e)),
        value_(detail::make_unique<T>(completion_.stream(), std::forward<U>(value)))
    {
    } // end future()

    __host__ __device__
    future(cudaStream_t s)
      : completion_(s)
    {
    } // end future()

    __host__ __device__
    future(future&& other)
      : completion_(std::move(other.completion_)),
        value_(std::move(other.value_))
    {
    } // end future()

    __host__ __device__
    future &operator=(future&& other)
    {
      completion_ = std::move(other.completion_);
      value_ = std::move(other.value_);
      return *this;
    } // end operator=()

    __host__ __device__
    void wait() const
    {
      completion_.wait();
    } // end wait()

    __host__ __device__
    T get() const
    {
      wait();

      return *value_;
    } // end get()

    __host__ __device__
    future<void> discard_value()
    {
      return std::move(completion_);
    } // end discard_value()

    __host__ __device__
    bool valid() const
    {
      return completion_.valid();
    } // end valid()

    // XXX only used by grid_executor
    //     think of a better way to expose this
    // XXX the existence of future_cast makes this superfluous i think
    __host__ __device__
    future<void>& void_future()
    {
      return completion_;
    } // end void_future()

    template<class U>
    __host__ __device__
    static future<T> make_ready(U&& value)
    {
      auto event = future<void>::make_ready();

      return future<T>{std::forward<U>(value), event};
    }

    // XXX this is only used by grid_executor::then_execute()
    __host__ __device__
    T* ptr()
    {
      return value_.get();
    }

    // XXX set_valid() should only be available to friends
    //     such as future<T> and grid_executor
    __host__ __device__
    void set_valid(cudaEvent_t event)
    {
      completion_.set_valid(event);
    }

  private:
    future<void> completion_;
    detail::unique_ptr<T> value_;
}; // end future<T>

inline __host__ __device__
future<void> make_ready_future()
{
  return future<void>::make_ready();
} // end make_ready_future()

template<class T>
inline __host__ __device__
future<typename std::decay<T>::type> make_ready_future(T&& value)
{
  return future<typename std::decay<T>::type>::make_ready(std::forward<T>(value));
} // end make_ready_future()

} // end namespace cuda
} // end namespace agency

<commit_msg>Tweak one of future<T>'s constructors<commit_after>/*
 *  Copyright 2008-2013 NVIDIA Corporation
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

#pragma once

#include <agency/detail/config.hpp>
#include <agency/cuda/detail/feature_test.hpp>
#include <agency/cuda/detail/terminate.hpp>
#include <agency/cuda/detail/unique_ptr.hpp>
#include <agency/cuda/detail/then_kernel.hpp>
#include <agency/cuda/detail/launch_kernel.hpp>
#include <agency/cuda/detail/workaround_unused_variable_warning.hpp>
#include <utility>

namespace agency
{
namespace cuda
{

template<typename T> class future;

template<>
class future<void>
{
  public:
    // XXX this should be private
    __host__ __device__
    future(cudaStream_t s) : future(s, 0) {}

    // XXX stream_ should default to per-thread default stream
    __host__ __device__
    future() : future(0) {}

    __host__ __device__
    future(future&& other)
      : future()
    {
      future::swap(stream_, other.stream_);
      future::swap(event_,  other.event_);
    } // end future()

    __host__ __device__
    future &operator=(future&& other)
    {
      future::swap(stream_, other.stream_);
      future::swap(event_,  other.event_);
      return *this;
    } // end operator=()

    __host__ __device__
    ~future()
    {
      if(valid())
      {
#if __cuda_lib_has_cudart
        // swallow errors
        cudaError_t e = cudaEventDestroy(event_);

#if __cuda_lib_has_printf
        if(e)
        {
          printf("CUDA error after cudaEventDestroy in agency::cuda::future<void> dtor: %s", cudaGetErrorString(e));
        } // end if
#endif // __cuda_lib_has_printf
#endif // __cuda_lib_has_cudart
      } // end if
    } // end ~future()

    __host__ __device__
    void wait() const
    {
      // XXX should probably check for valid() here

#if __cuda_lib_has_cudart

#ifndef __CUDA_ARCH__
      // XXX need to capture the error as an exception and then throw it in .get()
      detail::throw_on_error(cudaEventSynchronize(event_), "cudaEventSynchronize in agency::cuda<void>::future::wait");
#else
      // XXX need to capture the error as an exception and then throw it in .get()
      detail::throw_on_error(cudaDeviceSynchronize(), "cudaDeviceSynchronize in agency::cuda<void>::future::wait");
#endif // __CUDA_ARCH__

#else
      detail::terminate_with_message("agency::cuda::future<void>::wait() requires CUDART");
#endif // __cuda_lib_has_cudart
    } // end wait()

    __host__ __device__
    void get() const
    {
      wait();
    } // end get()

    // XXX we can eliminate this I think
    __host__ __device__
    future<void> discard_value()
    {
      return std::move(*this);
    } // end discard_value()

    __host__ __device__
    bool valid() const
    {
      return event_ != 0;
    } // end valid()

    __host__ __device__
    cudaEvent_t event() const
    {
      return event_;
    } // end event()

    __host__ __device__
    cudaStream_t stream() const
    {
      return stream_;
    } // end stream()

    __host__ __device__
    static future<void> make_ready()
    {
      cudaEvent_t ready_event = 0;

#if __cuda_lib_has_cudart
      detail::throw_on_error(cudaEventCreateWithFlags(&ready_event, event_create_flags), "cudaEventCreateWithFlags in agency::cuda::future<void>::make_ready");
#else
      detail::terminate_with_message("agency::cuda::future<void>::make_ready() requires CUDART");
#endif

      future<void> result;
      result.set_valid(ready_event);

      return result;
    }

    // XXX this is only used by grid_executor::then_execute()
    __host__ __device__
    std::nullptr_t ptr()
    {
      return nullptr;
    }

    template<class Function>
    __host__ __device__
    future<typename std::result_of<Function()>::type>
      then(Function f)
    {
      void (*kernel_ptr)(detail::my_nullptr_t, Function) = detail::then_kernel<Function>;
      detail::workaround_unused_variable_warning(kernel_ptr);

      using result_type = typename std::result_of<Function()>::type;

      using result_future_type = future<result_type>;

      result_future_type result(stream());

      cudaEvent_t next_event = detail::checked_launch_kernel_after_event_returning_next_event(reinterpret_cast<void*>(kernel_ptr), dim3{1}, dim3{1}, 0, stream(), event(), ptr(), f);

      // give next_event to the result future
      result.set_valid(result);

      return result;
    }

    // XXX set_valid() should only be available to friends
    //     such as future<T> and grid_executor
    __host__ __device__
    void set_valid(cudaEvent_t e)
    {
      event_ = e;
    }

  private:
    __host__ __device__
    future(cudaStream_t s, cudaEvent_t e) : stream_(s), event_(e) {}

    // implement swap to avoid depending on thrust::swap
    template<class T>
    __host__ __device__
    static void swap(T& a, T& b)
    {
      T tmp{a};
      a = b;
      b = tmp;
    }

    static const int event_create_flags = cudaEventDisableTiming;

    cudaStream_t stream_;
    cudaEvent_t event_;
}; // end future<void>

template<class T>
class future
{
  public:
    __host__ __device__
    future()
      : completion_()
    {}

    template<class U>
    __host__ __device__
    future(U&& value)
      : completion_(future<void>::make_ready()),
        value_(detail::make_unique<T>(completion_.stream(), std::forward<U>(value)))
    {
    } // end future()

    __host__ __device__
    future(cudaStream_t s)
      : completion_(s)
    {
    } // end future()

    __host__ __device__
    future(future&& other)
      : completion_(std::move(other.completion_)),
        value_(std::move(other.value_))
    {
    } // end future()

    __host__ __device__
    future &operator=(future&& other)
    {
      completion_ = std::move(other.completion_);
      value_ = std::move(other.value_);
      return *this;
    } // end operator=()

    __host__ __device__
    void wait() const
    {
      completion_.wait();
    } // end wait()

    __host__ __device__
    T get() const
    {
      wait();

      return *value_;
    } // end get()

    __host__ __device__
    future<void> discard_value()
    {
      return std::move(completion_);
    } // end discard_value()

    __host__ __device__
    bool valid() const
    {
      return completion_.valid();
    } // end valid()

    // XXX only used by grid_executor
    //     think of a better way to expose this
    // XXX the existence of future_cast makes this superfluous i think
    __host__ __device__
    future<void>& void_future()
    {
      return completion_;
    } // end void_future()

    template<class U>
    __host__ __device__
    static future<T> make_ready(U&& value)
    {
      return future<T>(std::forward<U>(value));
    }

    // XXX this is only used by grid_executor::then_execute()
    __host__ __device__
    T* ptr()
    {
      return value_.get();
    }

    // XXX set_valid() should only be available to friends
    //     such as future<T> and grid_executor
    __host__ __device__
    void set_valid(cudaEvent_t event)
    {
      completion_.set_valid(event);
    }

  private:
    future<void> completion_;
    detail::unique_ptr<T> value_;
}; // end future<T>

inline __host__ __device__
future<void> make_ready_future()
{
  return future<void>::make_ready();
} // end make_ready_future()

template<class T>
inline __host__ __device__
future<typename std::decay<T>::type> make_ready_future(T&& value)
{
  return future<typename std::decay<T>::type>::make_ready(std::forward<T>(value));
} // end make_ready_future()

} // end namespace cuda
} // end namespace agency

<|endoftext|>


<commit_before>#include "EnvironmentImpl.h"
#include "Session.h"
#include <cpuid.h>

namespace SubutaiLauncher
{

    const std::string EnvironmentImpl::EXTRA_PATH="/usr/local/bin:";

    EnvironmentImpl::EnvironmentImpl()
    {
        _logger = &Poco::Logger::get("subutai");
        _logger->trace("Starting new Environment instance");
    }

    EnvironmentImpl::~EnvironmentImpl()
    {
        _logger->trace("Environment::~Environment");
    }

    unsigned EnvironmentImpl::processorNum()
    {
        _logger->trace("Environment: Get CPU number");
        return Poco::Environment::processorCount();
    }

    unsigned EnvironmentImpl::is64()
    {
        _logger->trace("Environment: Determining architecture");
#if ( __WORDSIZE == 64 )
#define BUILD_64 1

#ifdef BUILD_64
        return 1;
#else
        return 0;
#endif
    }

    ULORAMSIZE_T EnvironmentImpl::ramSize()
    {
        _logger->debug("Environment: Retrieving RAM size");
        struct sysinfo info;
        _logger->trace("Running sysinfo");
        int rc = sysinfo(&info);
        if (rc == 0)
        {
            _logger->debug("Total mem size: %lu", info.totalram);
            return info.totalram;
        }
        return 0;
    }

    unsigned EnvironmentImpl::versionVBox()
    {

#if ( __WORDSIZE == 64 )
#define BUILD_64 1

#ifdef BUILD_64
        return 1;
#else
        return 0;
#endif
    }

    bool EnvironmentImpl::vtxEnabled()
    {
        Poco::Process::Args pArgs;
        pArgs.push_back("/proc/cpuinfo");
        Poco::Pipe pOut;
        Poco::ProcessHandle ph = Poco::Process::launch("/bin/cat", pArgs, 0, &pOut, 0);
        int rc = ph.wait();
        if (rc != 0)
        {
            _logger->error("Failed to cat /proc/cpuinfo. VTX query failed");
            return false;
        }
        std::string pBuffer;
        Poco::PipeInputStream istr(pOut);
        Poco::StreamCopier::copyToString(istr, pBuffer);
        Poco::StringTokenizer lines(pBuffer, " ", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY);
        for (auto it = lines.begin(); it != lines.end(); it++)
        {
            if ((*it) == "vme" || (*it) == "VME") return true;
        }
        return false;
    }

    std::string EnvironmentImpl::versionOS()
    {
        _logger->trace("Environment: Getting operating system information");
        std::string os;
        //os = Poco::Environment::osDisplayName() + " " + Poco::Environment::osVersion();
        os = Poco::Environment::osDisplayName();
        return os;
    }

    std::string EnvironmentImpl::versionNumber()
    {
        Poco::Process::Args pArgs;
        pArgs.push_back("/etc/lsb-release");
        Poco::Pipe pOut;
        Poco::ProcessHandle pH = Poco::Process::launch("/bin/cat", pArgs, 0, &pOut, 0);
        pH.wait();
        std::string pBuffer;
        Poco::PipeInputStream istr(pOut);
        Poco::StreamCopier::copyToString(istr, pBuffer);
        Poco::StringTokenizer lines(pBuffer, "\n", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY);
        for (auto it = lines.begin(); it != lines.end(); it++)
        {
            if ((*it).substr(0, 15) == "DISTRIB_RELEASE")
            {
                _logger->trace("Found DISTRIB_RELEASE: %s", (*it));
                std::string num = (*it).substr(16, 5);
                _logger->information("Extracted version: %s", num);
                return num;
            }
        }
        return "Unknown Version";
    }

    std::string EnvironmentImpl::cpuArch()
    {
        _logger->trace("Environment: Getting OS Architecture");
        std::string ar("");
        ar = Poco::Environment::osArchitecture();
        return ar;
    }

    unsigned int EnvironmentImpl::cpuNum()
    {
        return Poco::Environment::processorCount();
    }

    std::string EnvironmentImpl::getVar(const std::string& name, const std::string& defaultValue)
    {
        return Poco::Environment::get(name, defaultValue);
    }

    std::string EnvironmentImpl::setVar(const std::string& name, const std::string& value)
    {
        Poco::Environment::set(name, value);
        return value;
    }

    std::string EnvironmentImpl::getDefaultGateway()
    {
        std::string binary, gatewayName;
        int elnum;
        binary = "/bin/netstat";
        gatewayName = "0.0.0.0";
        elnum = 8;

        Poco::Process::Args args;
        args.push_back("-rn");

        Poco::Pipe pOut;
        Poco::ProcessHandle ph = Poco::Process::launch(binary, args, 0, &pOut, 0);
        ph.wait();

        Poco::PipeInputStream istr(pOut);
        std::string netstat;
        Poco::StreamCopier::copyToString(istr, netstat);

        Poco::StringTokenizer lines(netstat, "\n", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY);
        bool isDefault = false;
        for (auto line = lines.begin(); line != lines.end(); line++)
        {
            Poco::StringTokenizer elements((*line), " ", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY);
            int i = 0;
            for (auto el = elements.begin(); el != elements.end(); el++)
            {
                i++;
                if ((*el) == gatewayName) isDefault = true;
                if (isDefault && i == elnum) return (*el);
            }
        }
        return "unknown";
    }

    // Check whether NSSM tool is available
    bool EnvironmentImpl::isNSSMInstalled()
    {
        return false;
    }

    bool EnvironmentImpl::registerService(const std::string& name, const std::string& path, std::vector<std::string> args)
    {
        return false;
    }

    bool EnvironmentImpl::unregisterService(const std::string & name)
    {
        return false;
    }

    bool EnvironmentImpl::startService(const std::string& name)
    {
        return false;
    }

    bool EnvironmentImpl::stopService(const std::string& name)
    {
        return false;
    }

    void EnvironmentImpl::CreateShortcut(const std::string& source, const std::string& name)
    {

    }

    int32_t EnvironmentImpl::updatePath(const std::string& path)
    {
        return 0;
    }

    bool EnvironmentImpl::killProcess(const std::string & name)
    {
        return false;
    }

    std::string EnvironmentImpl::getDesktopDirectory()
    {
        return "";
    }

    bool EnvironmentImpl::isVBoxInstalled()
    {
        return false;
    }

    bool EnvironmentImpl::writeE2ERegistry(const std::string & name)
    {
        return false;
    }

    BOOL EnvironmentImpl::terminateWinProcess(DWORD dwProcessId, UINT uExitCode)
    {
        return false;
    }

    const std::string& EnvironmentImpl::getNetworkConfiguration() const
    {
        Poco::Process::Args pArgs;
        pArgs.push_back("addr");
        Poco::Pipe pOut;
        Poco::ProcessHandle ph = Poco::Process::launch("/usr/bin/ip", pArgs, 0, &pOut, 0);
        ph.wait();
        std::string pBuffer;
        Poco::PipeInputStream istr(pOut);
        Poco::StreamCopier::copyToString(istr, pBuffer);
        return pBuffer;
    }

    const std::string& EnvironmentImpl::getNetstat() const
    {
        Poco::Process::Args pArgs;
        pArgs.push_back("-rn");
        Poco::Pipe pOut;
        Poco::ProcessHandle ph = Poco::Process::launch("/bin/netstat", pArgs, 0, &pOut, 0);
        ph.wait();
        std::string pBuffer;
        Poco::PipeInputStream istr(pOut);
        Poco::StreamCopier::copyToString(istr, pBuffer);
        return pBuffer;
    }

    const std::string& EnvironmentImpl::getSystemInfo() const
    {
        return "";
    }

}
<commit_msg>Fixed linux build<commit_after>#include "EnvironmentImpl.h"
#include "Session.h"
#include <cpuid.h>

namespace SubutaiLauncher
{

    const std::string EnvironmentImpl::EXTRA_PATH="/usr/local/bin:";

    EnvironmentImpl::EnvironmentImpl()
    {
        _logger = &Poco::Logger::get("subutai");
        _logger->trace("Starting new Environment instance");
    }

    EnvironmentImpl::~EnvironmentImpl()
    {
        _logger->trace("Environment::~Environment");
    }

    unsigned EnvironmentImpl::processorNum()
    {
        _logger->trace("Environment: Get CPU number");
        return Poco::Environment::processorCount();
    }

    unsigned EnvironmentImpl::is64()
    {
        _logger->trace("Environment: Determining architecture");
        return 1;
    }

    ULORAMSIZE_T EnvironmentImpl::ramSize()
    {
        _logger->debug("Environment: Retrieving RAM size");
        struct sysinfo info;
        _logger->trace("Running sysinfo");
        int rc = sysinfo(&info);
        if (rc == 0)
        {
            _logger->debug("Total mem size: %lu", info.totalram);
            return info.totalram;
        }
        return 0;
    }

    unsigned EnvironmentImpl::versionVBox()
    {
        return 1;
    }

    bool EnvironmentImpl::vtxEnabled()
    {
        Poco::Process::Args pArgs;
        pArgs.push_back("/proc/cpuinfo");
        Poco::Pipe pOut;
        Poco::ProcessHandle ph = Poco::Process::launch("/bin/cat", pArgs, 0, &pOut, 0);
        int rc = ph.wait();
        if (rc != 0)
        {
            _logger->error("Failed to cat /proc/cpuinfo. VTX query failed");
            return false;
        }
        std::string pBuffer;
        Poco::PipeInputStream istr(pOut);
        Poco::StreamCopier::copyToString(istr, pBuffer);
        Poco::StringTokenizer lines(pBuffer, " ", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY);
        for (auto it = lines.begin(); it != lines.end(); it++)
        {
            if ((*it) == "vme" || (*it) == "VME") return true;
        }
        return false;
    }

    std::string EnvironmentImpl::versionOS()
    {
        _logger->trace("Environment: Getting operating system information");
        std::string os;
        //os = Poco::Environment::osDisplayName() + " " + Poco::Environment::osVersion();
        os = Poco::Environment::osDisplayName();
        return os;
    }

    std::string EnvironmentImpl::versionNumber()
    {
        Poco::Process::Args pArgs;
        pArgs.push_back("/etc/lsb-release");
        Poco::Pipe pOut;
        Poco::ProcessHandle pH = Poco::Process::launch("/bin/cat", pArgs, 0, &pOut, 0);
        pH.wait();
        std::string pBuffer;
        Poco::PipeInputStream istr(pOut);
        Poco::StreamCopier::copyToString(istr, pBuffer);
        Poco::StringTokenizer lines(pBuffer, "\n", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY);
        for (auto it = lines.begin(); it != lines.end(); it++)
        {
            if ((*it).substr(0, 15) == "DISTRIB_RELEASE")
            {
                _logger->trace("Found DISTRIB_RELEASE: %s", (*it));
                std::string num = (*it).substr(16, 5);
                _logger->information("Extracted version: %s", num);
                return num;
            }
        }
        return "Unknown Version";
    }

    std::string EnvironmentImpl::cpuArch()
    {
        _logger->trace("Environment: Getting OS Architecture");
        std::string ar("");
        ar = Poco::Environment::osArchitecture();
        return ar;
    }

    unsigned int EnvironmentImpl::cpuNum()
    {
        return Poco::Environment::processorCount();
    }

    std::string EnvironmentImpl::getVar(const std::string& name, const std::string& defaultValue)
    {
        return Poco::Environment::get(name, defaultValue);
    }

    std::string EnvironmentImpl::setVar(const std::string& name, const std::string& value)
    {
        Poco::Environment::set(name, value);
        return value;
    }

    std::string EnvironmentImpl::getDefaultGateway()
    {
        std::string binary, gatewayName;
        int elnum;
        binary = "/bin/netstat";
        gatewayName = "0.0.0.0";
        elnum = 8;

        Poco::Process::Args args;
        args.push_back("-rn");

        Poco::Pipe pOut;
        Poco::ProcessHandle ph = Poco::Process::launch(binary, args, 0, &pOut, 0);
        ph.wait();

        Poco::PipeInputStream istr(pOut);
        std::string netstat;
        Poco::StreamCopier::copyToString(istr, netstat);

        Poco::StringTokenizer lines(netstat, "\n", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY);
        bool isDefault = false;
        for (auto line = lines.begin(); line != lines.end(); line++)
        {
            Poco::StringTokenizer elements((*line), " ", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY);
            int i = 0;
            for (auto el = elements.begin(); el != elements.end(); el++)
            {
                i++;
                if ((*el) == gatewayName) isDefault = true;
                if (isDefault && i == elnum) return (*el);
            }
        }
        return "unknown";
    }

    // Check whether NSSM tool is available
    bool EnvironmentImpl::isNSSMInstalled()
    {
        return false;
    }

    bool EnvironmentImpl::registerService(const std::string& name, const std::string& path, std::vector<std::string> args)
    {
        return false;
    }

    bool EnvironmentImpl::unregisterService(const std::string & name)
    {
        return false;
    }

    bool EnvironmentImpl::startService(const std::string& name)
    {
        return false;
    }

    bool EnvironmentImpl::stopService(const std::string& name)
    {
        return false;
    }

    void EnvironmentImpl::CreateShortcut(const std::string& source, const std::string& name)
    {

    }

    int32_t EnvironmentImpl::updatePath(const std::string& path)
    {
        return 0;
    }

    bool EnvironmentImpl::killProcess(const std::string & name)
    {
        return false;
    }

    std::string EnvironmentImpl::getDesktopDirectory()
    {
        return "";
    }

    bool EnvironmentImpl::isVBoxInstalled()
    {
        return false;
    }

    bool EnvironmentImpl::writeE2ERegistry(const std::string & name)
    {
        return false;
    }

    BOOL EnvironmentImpl::terminateWinProcess(DWORD dwProcessId, UINT uExitCode)
    {
        return false;
    }

    const std::string& EnvironmentImpl::getNetworkConfiguration() const
    {
        Poco::Process::Args pArgs;
        pArgs.push_back("addr");
        Poco::Pipe pOut;
        Poco::ProcessHandle ph = Poco::Process::launch("/usr/bin/ip", pArgs, 0, &pOut, 0);
        ph.wait();
        std::string pBuffer;
        Poco::PipeInputStream istr(pOut);
        Poco::StreamCopier::copyToString(istr, pBuffer);
        return pBuffer;
    }

    const std::string& EnvironmentImpl::getNetstat() const
    {
        Poco::Process::Args pArgs;
        pArgs.push_back("-rn");
        Poco::Pipe pOut;
        Poco::ProcessHandle ph = Poco::Process::launch("/bin/netstat", pArgs, 0, &pOut, 0);
        ph.wait();
        std::string pBuffer;
        Poco::PipeInputStream istr(pOut);
        Poco::StreamCopier::copyToString(istr, pBuffer);
        return pBuffer;
    }

    const std::string& EnvironmentImpl::getSystemInfo() const
    {
        return "";
    }

}
<|endoftext|>


<commit_before>#include "GarageDoor/Filesystem.h"

std::vector<std::string> GarageDoor::Filesystem::ListDirectory(tstring path)
{
    std::vector<std::string> dirlist;

    DIR * dir = opendir(path.c_str());
    struct dirent * entry = readdir(dir);

    while (entry != NULL)
    {
        std::stringstream ss;
        ss << entry->d_name;
        std::string name = ss.str();

        if ( name == "." || name == ".." )
        {
            continue;
        }

        dirlist.push_back(name);
        entry = readdir(dir);
    }

    closedir(dir);
    return dirlist;
}
<commit_msg>Sick shit<commit_after>#include "GarageDoor/Filesystem.h"

std::vector<std::string> GarageDoor::Filesystem::ListDirectory(tstring path)
{
    std::vector<std::string> dirlist;

    DIR * dir = opendir(path.c_str());
    struct dirent * entry = readdir(dir);

    while (entry != NULL)
    {
        std::stringstream ss;
        ss << entry->d_name;
        std::string name = ss.str();

        if ( name != "." && name != ".." )
        {
            dirlist.push_back(name);
        }

        entry = readdir(dir);
    }

    closedir(dir);
    return dirlist;
}
<|endoftext|>


<commit_before>#include "BootConfig.hpp"

using namespace HomieInternals;

BootConfig::BootConfig()
: Boot("config")
, _http(80)
, _ssidCount(0)
, _wifiScanAvailable(false)
, _lastWifiScanEnded(true)
, _jsonWifiNetworks()
, _flaggedForReboot(false)
, _flaggedForRebootAt(0)
{
  this->_wifiScanTimer.setInterval(CONFIG_SCAN_INTERVAL);
}

BootConfig::~BootConfig() {
}

void BootConfig::setup() {
  Boot::setup();

  if (this->_interface->led.enabled) {
    digitalWrite(this->_interface->led.pin, this->_interface->led.on);
  }

  const char* deviceId = Helpers::getDeviceId();

  this->_interface->logger->log(F("Device ID is "));
  this->_interface->logger->logln(deviceId);

  WiFi.mode(WIFI_AP);

  char apName[MAX_WIFI_SSID_LENGTH];
  strcpy(apName, this->_interface->brand);
  strcat_P(apName, PSTR("-"));
  strcat(apName, Helpers::getDeviceId());

  WiFi.softAPConfig(ACCESS_POINT_IP, ACCESS_POINT_IP, IPAddress(255, 255, 255, 0));
  WiFi.softAP(apName, deviceId);

  this->_interface->logger->log(F("AP started as "));
  this->_interface->logger->logln(apName);

  this->_dns.setTTL(300);
  this->_dns.setErrorReplyCode(DNSReplyCode::ServerFailure);
  this->_dns.start(53, F("homie.config"), ACCESS_POINT_IP);

  this->_http.on("/", HTTP_GET, [this]() {
    this->_interface->logger->logln(F("Received index request"));
    this->_http.send(200, F("text/plain"), F("See Configuration API usage: http://marvinroger.viewdocs.io/homie-esp8266/6.-Configuration-API"));
  });
  this->_http.on("/heart", HTTP_GET, [this]() {
    this->_interface->logger->logln(F("Received heart request"));
    this->_http.send(200, FPSTR(PROGMEM_CONFIG_APPLICATION_JSON), F("{\"heart\":\"beat\"}"));
  });
  this->_http.on("/device-info", HTTP_GET, std::bind(&BootConfig::_onDeviceInfoRequest, this));
  this->_http.on("/networks", HTTP_GET, std::bind(&BootConfig::_onNetworksRequest, this));
  this->_http.on("/config", HTTP_PUT, std::bind(&BootConfig::_onConfigRequest, this));
  this->_http.on("/config", HTTP_OPTIONS, [this]() { // CORS
    this->_interface->logger->logln(F("Received CORS request for /config"));
    this->_http.sendContent(FPSTR(PROGMEM_CONFIG_CORS));
  });
  this->_http.begin();
}

void BootConfig::_generateNetworksJson() {
  DynamicJsonBuffer generatedJsonBuffer = DynamicJsonBuffer(JSON_OBJECT_SIZE(1) + JSON_ARRAY_SIZE(this->_ssidCount) + (this->_ssidCount * JSON_OBJECT_SIZE(3))); // 1 at root, 3 in childrend
  JsonObject& json = generatedJsonBuffer.createObject();

  int jsonLength = 15; // {"networks":[]}
  JsonArray& networks = json.createNestedArray("networks");
  for (int network = 0; network < this->_ssidCount; network++) {
    jsonLength += 36; // {"ssid":"","rssi":,"encryption":""},
    JsonObject& jsonNetwork = generatedJsonBuffer.createObject();
    jsonLength += WiFi.SSID(network).length();
    jsonNetwork["ssid"] = WiFi.SSID(network);
    jsonLength += 4;
    jsonNetwork["rssi"] = WiFi.RSSI(network);
    jsonLength += 4;
    switch (WiFi.encryptionType(network)) {
      case ENC_TYPE_WEP:
        jsonNetwork["encryption"] = "wep";
        break;
      case ENC_TYPE_TKIP:
        jsonNetwork["encryption"] = "wpa";
        break;
      case ENC_TYPE_CCMP:
        jsonNetwork["encryption"] = "wpa2";
        break;
      case ENC_TYPE_NONE:
        jsonNetwork["encryption"] = "none";
        break;
      case ENC_TYPE_AUTO:
        jsonNetwork["encryption"] = "auto";
        break;
    }

    networks.add(jsonNetwork);
  }

  jsonLength++; // \0

  delete[] this->_jsonWifiNetworks;
  this->_jsonWifiNetworks = new char[jsonLength];
  json.printTo(this->_jsonWifiNetworks, jsonLength);
}

void BootConfig::_onDeviceInfoRequest() {
  this->_interface->logger->logln(F("Received device info request"));

  DynamicJsonBuffer jsonBuffer = DynamicJsonBuffer(JSON_OBJECT_SIZE(4) + JSON_OBJECT_SIZE(2) + JSON_ARRAY_SIZE(this->_interface->registeredNodesCount) + (this->_interface->registeredNodesCount * JSON_OBJECT_SIZE(2)));
  int jsonLength = 82; // {"device_id":"","homie_version":"","firmware":{"name":"","version":""},"nodes":[]}
  JsonObject& json = jsonBuffer.createObject();
  jsonLength += strlen(Helpers::getDeviceId());
  json["device_id"] = Helpers::getDeviceId();
  jsonLength += strlen(VERSION);
  json["homie_version"] = VERSION;
  JsonObject& firmware = json.createNestedObject("firmware");
  jsonLength += strlen(this->_interface->firmware.name);
  firmware["name"] = this->_interface->firmware.name;
  jsonLength += strlen(this->_interface->firmware.version);
  firmware["version"] = this->_interface->firmware.version;

  JsonArray& nodes = json.createNestedArray("nodes");
  for (int i = 0; i < this->_interface->registeredNodesCount; i++) {
    jsonLength += 20; // {"id":"","type":""},
    const HomieNode* node = this->_interface->registeredNodes[i];
    JsonObject& jsonNode = jsonBuffer.createObject();
    jsonLength += strlen(node->getId());
    jsonNode["id"] = node->getId();
    jsonLength += strlen(node->getType());
    jsonNode["type"] = node->getType();

    nodes.add(jsonNode);
  }

  jsonLength++; // \0

  std::unique_ptr<char[]> jsonString(new char[jsonLength]);
  json.printTo(jsonString.get(), jsonLength);
  this->_http.send(200, FPSTR(PROGMEM_CONFIG_APPLICATION_JSON), jsonString.get());
}

void BootConfig::_onNetworksRequest() {
  this->_interface->logger->logln(F("Received networks request"));
  if (this->_wifiScanAvailable) {
    this->_http.send(200, FPSTR(PROGMEM_CONFIG_APPLICATION_JSON), this->_jsonWifiNetworks);
  } else {
    this->_http.send(503, FPSTR(PROGMEM_CONFIG_APPLICATION_JSON), FPSTR(PROGMEM_CONFIG_NETWORKS_FAILURE));
  }
}

void BootConfig::_onConfigRequest() {
  this->_interface->logger->logln(F("Received config request"));
  if (this->_flaggedForReboot) {
    this->_interface->logger->logln(F("✖ Device already configured"));
    String errorJson = String(FPSTR(PROGMEM_CONFIG_JSON_FAILURE_BEGINNING));
    errorJson.concat(F("Device already configured\"}"));
    this->_http.send(403, FPSTR(PROGMEM_CONFIG_APPLICATION_JSON), errorJson);
    return;
  }

  StaticJsonBuffer<MAX_JSON_CONFIG_ARDUINOJSON_BUFFER_SIZE> parseJsonBuffer;
  char* bodyCharArray = strdup(this->_http.arg("plain").c_str());
  JsonObject& parsedJson = parseJsonBuffer.parseObject(bodyCharArray); // do not use plain String, else fails
  if (!parsedJson.success()) {
    this->_interface->logger->logln(F("✖ Invalid or too big JSON"));
    String errorJson = String(FPSTR(PROGMEM_CONFIG_JSON_FAILURE_BEGINNING));
    errorJson.concat(F("Invalid or too big JSON\"}"));
    this->_http.send(400, FPSTR(PROGMEM_CONFIG_APPLICATION_JSON), errorJson);
    return;
  }

  ConfigValidationResult configValidationResult = Helpers::validateConfig(parsedJson);
  free(bodyCharArray);
  if (!configValidationResult.valid) {
    this->_interface->logger->log(F("✖ Config file is not valid, reason: "));
    this->_interface->logger->logln(configValidationResult.reason);
    String errorJson = String(FPSTR(PROGMEM_CONFIG_JSON_FAILURE_BEGINNING));
    errorJson.concat(F("Config file is not valid, reason: "));
    errorJson.concat(configValidationResult.reason);
    errorJson.concat(F("\"}"));
    this->_http.send(400, FPSTR(PROGMEM_CONFIG_APPLICATION_JSON), errorJson);
    return;
  }

  this->_interface->config->write(this->_http.arg("plain"));

  this->_interface->logger->logln(F("✔ Configured"));

  this->_http.send(200, FPSTR(PROGMEM_CONFIG_APPLICATION_JSON), F("{\"success\":true}"));

  this->_flaggedForReboot = true; // We don't reboot immediately, otherwise the response above is not sent
  this->_flaggedForRebootAt = millis();
}

void BootConfig::loop() {
  Boot::loop();

  this->_dns.processNextRequest();
  this->_http.handleClient();

  if (this->_flaggedForReboot) {
    if (millis() - this->_flaggedForRebootAt >= 3000UL) {
      this->_interface->logger->logln(F("↻ Rebooting into normal mode..."));
      ESP.restart();
    }

    return;
  }

  if (!this->_lastWifiScanEnded) {
    int8_t scanResult = WiFi.scanComplete();

    switch (scanResult) {
      case WIFI_SCAN_RUNNING:
        return;
      case WIFI_SCAN_FAILED:
        this->_interface->logger->logln(F("✖ Wi-Fi scan failed"));
        this->_ssidCount = 0;
        this->_wifiScanTimer.reset();
        break;
      default:
        this->_interface->logger->logln(F("✔ Wi-Fi scan completed"));
        this->_ssidCount = scanResult;
        this->_generateNetworksJson();
        this->_wifiScanAvailable = true;
        break;
    }

    this->_lastWifiScanEnded = true;
  }

  if (this->_lastWifiScanEnded && this->_wifiScanTimer.check()) {
    this->_interface->logger->logln(F("Triggering Wi-Fi scan..."));
    WiFi.scanNetworks(true);
    this->_wifiScanTimer.tick();
    this->_lastWifiScanEnded = false;
  }
}
<commit_msg>:bug: Fix compatibility with ArduinoJson 5.11.0 (#362)<commit_after>#include "BootConfig.hpp"

using namespace HomieInternals;

BootConfig::BootConfig()
: Boot("config")
, _http(80)
, _ssidCount(0)
, _wifiScanAvailable(false)
, _lastWifiScanEnded(true)
, _jsonWifiNetworks()
, _flaggedForReboot(false)
, _flaggedForRebootAt(0)
{
  this->_wifiScanTimer.setInterval(CONFIG_SCAN_INTERVAL);
}

BootConfig::~BootConfig() {
}

void BootConfig::setup() {
  Boot::setup();

  if (this->_interface->led.enabled) {
    digitalWrite(this->_interface->led.pin, this->_interface->led.on);
  }

  const char* deviceId = Helpers::getDeviceId();

  this->_interface->logger->log(F("Device ID is "));
  this->_interface->logger->logln(deviceId);

  WiFi.mode(WIFI_AP);

  char apName[MAX_WIFI_SSID_LENGTH];
  strcpy(apName, this->_interface->brand);
  strcat_P(apName, PSTR("-"));
  strcat(apName, Helpers::getDeviceId());

  WiFi.softAPConfig(ACCESS_POINT_IP, ACCESS_POINT_IP, IPAddress(255, 255, 255, 0));
  WiFi.softAP(apName, deviceId);

  this->_interface->logger->log(F("AP started as "));
  this->_interface->logger->logln(apName);

  this->_dns.setTTL(300);
  this->_dns.setErrorReplyCode(DNSReplyCode::ServerFailure);
  this->_dns.start(53, F("homie.config"), ACCESS_POINT_IP);

  this->_http.on("/", HTTP_GET, [this]() {
    this->_interface->logger->logln(F("Received index request"));
    this->_http.send(200, F("text/plain"), F("See Configuration API usage: http://marvinroger.viewdocs.io/homie-esp8266/6.-Configuration-API"));
  });
  this->_http.on("/heart", HTTP_GET, [this]() {
    this->_interface->logger->logln(F("Received heart request"));
    this->_http.send(200, FPSTR(PROGMEM_CONFIG_APPLICATION_JSON), F("{\"heart\":\"beat\"}"));
  });
  this->_http.on("/device-info", HTTP_GET, std::bind(&BootConfig::_onDeviceInfoRequest, this));
  this->_http.on("/networks", HTTP_GET, std::bind(&BootConfig::_onNetworksRequest, this));
  this->_http.on("/config", HTTP_PUT, std::bind(&BootConfig::_onConfigRequest, this));
  this->_http.on("/config", HTTP_OPTIONS, [this]() { // CORS
    this->_interface->logger->logln(F("Received CORS request for /config"));
    this->_http.sendContent(FPSTR(PROGMEM_CONFIG_CORS));
  });
  this->_http.begin();
}

void BootConfig::_generateNetworksJson() {
  DynamicJsonBuffer generatedJsonBuffer(JSON_OBJECT_SIZE(1) + JSON_ARRAY_SIZE(this->_ssidCount) + (this->_ssidCount * JSON_OBJECT_SIZE(3))); // 1 at root, 3 in childrend
  JsonObject& json = generatedJsonBuffer.createObject();

  int jsonLength = 15; // {"networks":[]}
  JsonArray& networks = json.createNestedArray("networks");
  for (int network = 0; network < this->_ssidCount; network++) {
    jsonLength += 36; // {"ssid":"","rssi":,"encryption":""},
    JsonObject& jsonNetwork = generatedJsonBuffer.createObject();
    jsonLength += WiFi.SSID(network).length();
    jsonNetwork["ssid"] = WiFi.SSID(network);
    jsonLength += 4;
    jsonNetwork["rssi"] = WiFi.RSSI(network);
    jsonLength += 4;
    switch (WiFi.encryptionType(network)) {
      case ENC_TYPE_WEP:
        jsonNetwork["encryption"] = "wep";
        break;
      case ENC_TYPE_TKIP:
        jsonNetwork["encryption"] = "wpa";
        break;
      case ENC_TYPE_CCMP:
        jsonNetwork["encryption"] = "wpa2";
        break;
      case ENC_TYPE_NONE:
        jsonNetwork["encryption"] = "none";
        break;
      case ENC_TYPE_AUTO:
        jsonNetwork["encryption"] = "auto";
        break;
    }

    networks.add(jsonNetwork);
  }

  jsonLength++; // \0

  delete[] this->_jsonWifiNetworks;
  this->_jsonWifiNetworks = new char[jsonLength];
  json.printTo(this->_jsonWifiNetworks, jsonLength);
}

void BootConfig::_onDeviceInfoRequest() {
  this->_interface->logger->logln(F("Received device info request"));

  DynamicJsonBuffer jsonBuffer(JSON_OBJECT_SIZE(4) + JSON_OBJECT_SIZE(2) + JSON_ARRAY_SIZE(this->_interface->registeredNodesCount) + (this->_interface->registeredNodesCount * JSON_OBJECT_SIZE(2)));
  int jsonLength = 82; // {"device_id":"","homie_version":"","firmware":{"name":"","version":""},"nodes":[]}
  JsonObject& json = jsonBuffer.createObject();
  jsonLength += strlen(Helpers::getDeviceId());
  json["device_id"] = Helpers::getDeviceId();
  jsonLength += strlen(VERSION);
  json["homie_version"] = VERSION;
  JsonObject& firmware = json.createNestedObject("firmware");
  jsonLength += strlen(this->_interface->firmware.name);
  firmware["name"] = this->_interface->firmware.name;
  jsonLength += strlen(this->_interface->firmware.version);
  firmware["version"] = this->_interface->firmware.version;

  JsonArray& nodes = json.createNestedArray("nodes");
  for (int i = 0; i < this->_interface->registeredNodesCount; i++) {
    jsonLength += 20; // {"id":"","type":""},
    const HomieNode* node = this->_interface->registeredNodes[i];
    JsonObject& jsonNode = jsonBuffer.createObject();
    jsonLength += strlen(node->getId());
    jsonNode["id"] = node->getId();
    jsonLength += strlen(node->getType());
    jsonNode["type"] = node->getType();

    nodes.add(jsonNode);
  }

  jsonLength++; // \0

  std::unique_ptr<char[]> jsonString(new char[jsonLength]);
  json.printTo(jsonString.get(), jsonLength);
  this->_http.send(200, FPSTR(PROGMEM_CONFIG_APPLICATION_JSON), jsonString.get());
}

void BootConfig::_onNetworksRequest() {
  this->_interface->logger->logln(F("Received networks request"));
  if (this->_wifiScanAvailable) {
    this->_http.send(200, FPSTR(PROGMEM_CONFIG_APPLICATION_JSON), this->_jsonWifiNetworks);
  } else {
    this->_http.send(503, FPSTR(PROGMEM_CONFIG_APPLICATION_JSON), FPSTR(PROGMEM_CONFIG_NETWORKS_FAILURE));
  }
}

void BootConfig::_onConfigRequest() {
  this->_interface->logger->logln(F("Received config request"));
  if (this->_flaggedForReboot) {
    this->_interface->logger->logln(F("✖ Device already configured"));
    String errorJson = String(FPSTR(PROGMEM_CONFIG_JSON_FAILURE_BEGINNING));
    errorJson.concat(F("Device already configured\"}"));
    this->_http.send(403, FPSTR(PROGMEM_CONFIG_APPLICATION_JSON), errorJson);
    return;
  }

  StaticJsonBuffer<MAX_JSON_CONFIG_ARDUINOJSON_BUFFER_SIZE> parseJsonBuffer;
  char* bodyCharArray = strdup(this->_http.arg("plain").c_str());
  JsonObject& parsedJson = parseJsonBuffer.parseObject(bodyCharArray); // do not use plain String, else fails
  if (!parsedJson.success()) {
    this->_interface->logger->logln(F("✖ Invalid or too big JSON"));
    String errorJson = String(FPSTR(PROGMEM_CONFIG_JSON_FAILURE_BEGINNING));
    errorJson.concat(F("Invalid or too big JSON\"}"));
    this->_http.send(400, FPSTR(PROGMEM_CONFIG_APPLICATION_JSON), errorJson);
    return;
  }

  ConfigValidationResult configValidationResult = Helpers::validateConfig(parsedJson);
  free(bodyCharArray);
  if (!configValidationResult.valid) {
    this->_interface->logger->log(F("✖ Config file is not valid, reason: "));
    this->_interface->logger->logln(configValidationResult.reason);
    String errorJson = String(FPSTR(PROGMEM_CONFIG_JSON_FAILURE_BEGINNING));
    errorJson.concat(F("Config file is not valid, reason: "));
    errorJson.concat(configValidationResult.reason);
    errorJson.concat(F("\"}"));
    this->_http.send(400, FPSTR(PROGMEM_CONFIG_APPLICATION_JSON), errorJson);
    return;
  }

  this->_interface->config->write(this->_http.arg("plain"));

  this->_interface->logger->logln(F("✔ Configured"));

  this->_http.send(200, FPSTR(PROGMEM_CONFIG_APPLICATION_JSON), F("{\"success\":true}"));

  this->_flaggedForReboot = true; // We don't reboot immediately, otherwise the response above is not sent
  this->_flaggedForRebootAt = millis();
}

void BootConfig::loop() {
  Boot::loop();

  this->_dns.processNextRequest();
  this->_http.handleClient();

  if (this->_flaggedForReboot) {
    if (millis() - this->_flaggedForRebootAt >= 3000UL) {
      this->_interface->logger->logln(F("↻ Rebooting into normal mode..."));
      ESP.restart();
    }

    return;
  }

  if (!this->_lastWifiScanEnded) {
    int8_t scanResult = WiFi.scanComplete();

    switch (scanResult) {
      case WIFI_SCAN_RUNNING:
        return;
      case WIFI_SCAN_FAILED:
        this->_interface->logger->logln(F("✖ Wi-Fi scan failed"));
        this->_ssidCount = 0;
        this->_wifiScanTimer.reset();
        break;
      default:
        this->_interface->logger->logln(F("✔ Wi-Fi scan completed"));
        this->_ssidCount = scanResult;
        this->_generateNetworksJson();
        this->_wifiScanAvailable = true;
        break;
    }

    this->_lastWifiScanEnded = true;
  }

  if (this->_lastWifiScanEnded && this->_wifiScanTimer.check()) {
    this->_interface->logger->logln(F("Triggering Wi-Fi scan..."));
    WiFi.scanNetworks(true);
    this->_wifiScanTimer.tick();
    this->_lastWifiScanEnded = false;
  }
}
<|endoftext|>


<commit_before>//////////////////////////////////////////////////////////////////////////
//
//  Copyright (c) 2007-2008, Image Engine Design Inc. All rights reserved.
//
//  Redistribution and use in source and binary forms, with or without
//  modification, are permitted provided that the following conditions are
//  met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
//     * Neither the name of Image Engine Design nor the names of any
//       other contributors to this software may be used to endorse or
//       promote products derived from this software without specific prior
//       written permission.
//
//  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
//  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
//  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
//  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
//  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
//  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
//  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
//  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
//  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
//  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
//  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
//////////////////////////////////////////////////////////////////////////

#include "IECore/EXRImageWriter.h"
#include "IECore/MessageHandler.h"
#include "IECore/VectorTypedData.h"
#include "IECore/ByteOrder.h"
#include "IECore/ImagePrimitive.h"
#include "IECore/FileNameParameter.h"
#include "IECore/BoxOps.h"

#include "boost/format.hpp"

#include <fstream>

using namespace IECore;

using std::string;
using std::vector;

using Imath::Box2i;
using namespace Imf;

const Writer::WriterDescription<EXRImageWriter> EXRImageWriter::m_writerDescription("exr");

EXRImageWriter::EXRImageWriter()
		: ImageWriter("EXRImageWriter", "Serializes images to the OpenEXR HDR image format")
{
}

EXRImageWriter::EXRImageWriter(ObjectPtr image, const string &fileName)
		: ImageWriter("EXRImageWriter", "Serializes images to the OpenEXR HDR image format" )
{
	assert( m_objectParameter );
	assert( m_fileNameParameter );

	m_objectParameter->setValue( image );
	m_fileNameParameter->setTypedValue( fileName );
}

// \todo "names" should be const
void EXRImageWriter::writeImage(vector<string> &names, ConstImagePrimitivePtr image, const Box2i &dataWindow)
{
	assert( image );

	// create the header
	int width  = 1 + boxSize( dataWindow ).x;
	int height = 1 + boxSize( dataWindow ).y;

	try
	{
		/// \todo Add parameters for compression, etc
		Header header(width, height, 1, Imath::V2f(0.0, 0.0), 1, INCREASING_Y, PIZ_COMPRESSION);
		header.dataWindow() = dataWindow;
		header.displayWindow() = image->getDisplayWindow();

		// create the framebuffer
		FrameBuffer fb;

		// add the channels into the header with the appropriate types
		vector<string>::const_iterator i = names.begin();
		for (vector<string>::const_iterator i = names.begin(); i != names.end(); ++i)
		{
			const char *name = (*i).c_str();

			// get the image channel
			PrimitiveVariableMap::const_iterator pit = image->variables.find(name);
			if ( pit == image->variables.end() )
			{
				throw IOException( ( boost::format("EXRImageWriter: Could not find image channel \"%s\"") % name ).str() );
			}

			ConstDataPtr channelData = pit->second.data;
			if (!channelData)
			{
				throw IOException( ( boost::format("EXRImageWriter: Channel \"%s\" has no data") % name ).str() );
			}

			switch (channelData->typeId())
			{
			case FloatVectorDataTypeId:
				writeTypedChannel<float>(name, image, dataWindow,
				                         boost::static_pointer_cast<const FloatVectorData>(channelData)->readable(),
				                         FLOAT, header, fb);
				break;

			case UIntVectorDataTypeId:
				writeTypedChannel<unsigned int>(name, image, dataWindow,
				                                boost::static_pointer_cast<const UIntVectorData>(channelData)->readable(),
				                                UINT, header, fb);
				break;

			case HalfVectorDataTypeId:
				writeTypedChannel<half>(name, image, dataWindow,
				                        boost::static_pointer_cast<const HalfVectorData>(channelData)->readable(),
				                        HALF, header, fb);
				break;

			default:
				throw IOException( ( boost::format("EXRImageWriter: Invalid data type \"%s\" for channel \"%s\"") % channelData->typeName() % name ).str() );
			}
		}

		// create the output file, write, implicitly close
		OutputFile out(fileName().c_str(), header);

		out.setFrameBuffer(fb);
		out.writePixels(height);
	}
	catch ( Exception &e )
	{
		throw;
	}
	catch ( std::exception &e )
	{
		throw IOException( ( boost::format("EXRImageWriter: %s") % e.what() ).str() );
	}
	catch ( ... )
	{
		throw IOException( "EXRImageWriter: Unexpected error" );
	}

}

template<typename T>
void EXRImageWriter::writeTypedChannel(const char *name, ConstImagePrimitivePtr image, const Box2i &dataWindow,
                                       const vector<T> &channel, const Imf::PixelType pixelType, Header &header, FrameBuffer &fb)
{
	assert( name );

	/// \todo Remove this unused parameter
	(void) image;

	int width = 1 + dataWindow.max.x - dataWindow.min.x;

	// update the header
	header.channels().insert( name, Channel(pixelType) );

	// update the framebuffer
	char *offset = (char *) (&channel[0] - (dataWindow.min.x + width * dataWindow.min.y));
	fb.insert(name, Slice(pixelType, offset, sizeof(T), sizeof(T) * width));
}
<commit_msg>Removed ununsed variable<commit_after>//////////////////////////////////////////////////////////////////////////
//
//  Copyright (c) 2007-2008, Image Engine Design Inc. All rights reserved.
//
//  Redistribution and use in source and binary forms, with or without
//  modification, are permitted provided that the following conditions are
//  met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
//     * Neither the name of Image Engine Design nor the names of any
//       other contributors to this software may be used to endorse or
//       promote products derived from this software without specific prior
//       written permission.
//
//  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
//  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
//  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
//  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
//  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
//  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
//  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
//  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
//  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
//  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
//  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
//////////////////////////////////////////////////////////////////////////

#include "IECore/EXRImageWriter.h"
#include "IECore/MessageHandler.h"
#include "IECore/VectorTypedData.h"
#include "IECore/ByteOrder.h"
#include "IECore/ImagePrimitive.h"
#include "IECore/FileNameParameter.h"
#include "IECore/BoxOps.h"

#include "boost/format.hpp"

#include <fstream>

using namespace IECore;

using std::string;
using std::vector;

using Imath::Box2i;
using namespace Imf;

const Writer::WriterDescription<EXRImageWriter> EXRImageWriter::m_writerDescription("exr");

EXRImageWriter::EXRImageWriter()
		: ImageWriter("EXRImageWriter", "Serializes images to the OpenEXR HDR image format")
{
}

EXRImageWriter::EXRImageWriter(ObjectPtr image, const string &fileName)
		: ImageWriter("EXRImageWriter", "Serializes images to the OpenEXR HDR image format" )
{
	assert( m_objectParameter );
	assert( m_fileNameParameter );

	m_objectParameter->setValue( image );
	m_fileNameParameter->setTypedValue( fileName );
}

// \todo "names" should be const
void EXRImageWriter::writeImage(vector<string> &names, ConstImagePrimitivePtr image, const Box2i &dataWindow)
{
	assert( image );

	// create the header
	int width  = 1 + boxSize( dataWindow ).x;
	int height = 1 + boxSize( dataWindow ).y;

	try
	{
		/// \todo Add parameters for compression, etc
		Header header(width, height, 1, Imath::V2f(0.0, 0.0), 1, INCREASING_Y, PIZ_COMPRESSION);
		header.dataWindow() = dataWindow;
		header.displayWindow() = image->getDisplayWindow();

		// create the framebuffer
		FrameBuffer fb;

		// add the channels into the header with the appropriate types
		for (vector<string>::const_iterator i = names.begin(); i != names.end(); ++i)
		{
			const char *name = (*i).c_str();

			// get the image channel
			PrimitiveVariableMap::const_iterator pit = image->variables.find(name);
			if ( pit == image->variables.end() )
			{
				throw IOException( ( boost::format("EXRImageWriter: Could not find image channel \"%s\"") % name ).str() );
			}

			ConstDataPtr channelData = pit->second.data;
			if (!channelData)
			{
				throw IOException( ( boost::format("EXRImageWriter: Channel \"%s\" has no data") % name ).str() );
			}

			switch (channelData->typeId())
			{
			case FloatVectorDataTypeId:
				writeTypedChannel<float>(name, image, dataWindow,
				                         boost::static_pointer_cast<const FloatVectorData>(channelData)->readable(),
				                         FLOAT, header, fb);
				break;

			case UIntVectorDataTypeId:
				writeTypedChannel<unsigned int>(name, image, dataWindow,
				                                boost::static_pointer_cast<const UIntVectorData>(channelData)->readable(),
				                                UINT, header, fb);
				break;

			case HalfVectorDataTypeId:
				writeTypedChannel<half>(name, image, dataWindow,
				                        boost::static_pointer_cast<const HalfVectorData>(channelData)->readable(),
				                        HALF, header, fb);
				break;

			default:
				throw IOException( ( boost::format("EXRImageWriter: Invalid data type \"%s\" for channel \"%s\"") % channelData->typeName() % name ).str() );
			}
		}

		// create the output file, write, implicitly close
		OutputFile out(fileName().c_str(), header);

		out.setFrameBuffer(fb);
		out.writePixels(height);
	}
	catch ( Exception &e )
	{
		throw;
	}
	catch ( std::exception &e )
	{
		throw IOException( ( boost::format("EXRImageWriter: %s") % e.what() ).str() );
	}
	catch ( ... )
	{
		throw IOException( "EXRImageWriter: Unexpected error" );
	}

}

template<typename T>
void EXRImageWriter::writeTypedChannel(const char *name, ConstImagePrimitivePtr image, const Box2i &dataWindow,
                                       const vector<T> &channel, const Imf::PixelType pixelType, Header &header, FrameBuffer &fb)
{
	assert( name );

	/// \todo Remove this unused parameter
	(void) image;

	int width = 1 + dataWindow.max.x - dataWindow.min.x;

	// update the header
	header.channels().insert( name, Channel(pixelType) );

	// update the framebuffer
	char *offset = (char *) (&channel[0] - (dataWindow.min.x + width * dataWindow.min.y));
	fb.insert(name, Slice(pixelType, offset, sizeof(T), sizeof(T) * width));
}
<|endoftext|>


<commit_before><commit_msg>comment<commit_after><|endoftext|>


<commit_before><commit_msg>am 8fc5a63d: Merge change 2431 into donut<commit_after><|endoftext|>


<commit_before><commit_msg>Core/Network: [Add] BaseUrlInterceptor class<commit_after><|endoftext|>


<commit_before>/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/data/root_dataset.h"

#include <functional>
#include <string>
#include <utility>

#include "tensorflow/core/data/dataset_utils.h"
#include "tensorflow/core/data/name_utils.h"
#include "tensorflow/core/data/rewrite_utils.h"
#include "tensorflow/core/framework/model.pb.h"
#include "tensorflow/core/platform/errors.h"
#include "tensorflow/core/platform/host_info.h"
#include "tensorflow/core/platform/refcount.h"
#include "tensorflow/core/platform/stringprintf.h"

namespace tensorflow {
namespace data {
namespace {

constexpr char kDatasetType[] = "Root";

constexpr char kAlgorithm[] = "algorithm";
constexpr char kCpuBudget[] = "cpu_budget";
constexpr char kExperiments[] = "experiments";
constexpr char kInjectPrefetchEligibleOpt[] = "inject_prefetch_eligible";
constexpr char kIntraOpParallelism[] = "intra_op_parallelism";
constexpr char kMemBandwidth[] = "mem_bw_used_megabytes_per_sec";
constexpr char kPrivateThreadpoolSize[] = "threadpool_size";
constexpr char kRamBudget[] = "ram_budget_megabytes";
constexpr char kRamUsage[] = "ram_usage_megabytes";
constexpr char kMaxBufferBytes[] = "max_buffered_megabytes";

// If value `x` matches `y`, returns default value `z`. Otherwise, return `x`.
inline int64_t value_or_default(int64_t x, int64_t y, int64_t z) {
  return x == y ? z : x;
}

void SetRootDatasetParams(const Options& options, RootDataset::Params* params) {
  if (ShouldConfigureMaxIntraOpParallelism(options)) {
    params->max_intra_op_parallelism =
        options.threading_options().max_intra_op_parallelism();
  }
  if (ShouldUsePrivateThreadPool(options)) {
    params->private_threadpool_size =
        options.threading_options().private_threadpool_size();
  }
  params->autotune = ShouldUseAutotuning(options);
  if (params->autotune) {
    params->autotune_algorithm =
        options.autotune_options().optional_autotune_algorithm_case() ==
                AutotuneOptions::kAutotuneAlgorithm
            ? options.autotune_options().autotune_algorithm()
            : model::AutotuneAlgorithm::DEFAULT;
    params->autotune_cpu_budget = value_or_default(
        options.autotune_options().cpu_budget(), 0, GetCpuBudget());
    params->autotune_ram_budget =
        value_or_default(options.autotune_options().ram_budget(), 0,
                         model::kRamBudgetShare * port::AvailableRam());
  }
}

void AddTraceMetadata(const RootDataset::Params& params,
                      TraceMeMetadata* trace_metadata) {
  if (params.autotune) {
    trace_metadata->push_back(std::make_pair(
        kAlgorithm, model::AutotuneAlgorithm_Name(params.autotune_algorithm)));
    trace_metadata->push_back(std::make_pair(
        kCpuBudget, strings::Printf("%lld", static_cast<long long>(
                                                params.autotune_cpu_budget))));
    trace_metadata->push_back(std::make_pair(
        kRamBudget,
        strings::Printf("%lld", static_cast<long long>(
                                    params.autotune_ram_budget / 1.0e6))));
  }
  if (params.max_intra_op_parallelism >= 0) {
    trace_metadata->push_back(std::make_pair(
        kIntraOpParallelism,
        strings::Printf("%lld", static_cast<long long>(value_or_default(
                                    params.max_intra_op_parallelism, 0,
                                    port::MaxParallelism())))));
  }
  if (params.private_threadpool_size >= 0) {
    trace_metadata->push_back(std::make_pair(
        kPrivateThreadpoolSize,
        strings::Printf("%lld", static_cast<long long>(value_or_default(
                                    params.private_threadpool_size, 0,
                                    port::MaxParallelism())))));
  }
  auto experiments = GetExperiments();
  if (!experiments.empty()) {
    trace_metadata->push_back(
        std::make_pair(kExperiments, absl::StrJoin(experiments, " ")));
  }
}
}  // namespace

// static
Status RootDataset::FromOptions(const DatasetBase* input,
                                DatasetBase** output) {
  Params params;
  SetRootDatasetParams(input->options(), &params);
  *output = new RootDataset(input, params);
  (*output)->Initialize(/*metadata=*/{});
  return OkStatus();
}

Status RootDataset::FromOptions(core::RefCountPtr<DatasetBase> input,
                                DatasetBase** output) {
  Params params;
  SetRootDatasetParams(input->options(), &params);
  *output = new RootDataset(std::move(input), params);
  (*output)->Initialize(/*metadata=*/{});
  return OkStatus();
}

class RootDataset::Iterator : public DatasetIterator<RootDataset> {
 public:
  explicit Iterator(const Params& params)
      : DatasetIterator<RootDataset>(params) {
    if (dataset()->params_.autotune) {
      model_ = std::make_shared<model::Model>();
    }
    if (dataset()->params_.max_intra_op_parallelism >= 0) {
      max_intra_op_parallelism_ =
          value_or_default(dataset()->params_.max_intra_op_parallelism, 0,
                           port::MaxParallelism());
    }
    if (dataset()->params_.private_threadpool_size >= 0) {
      threadpool_size_ =
          value_or_default(dataset()->params_.private_threadpool_size, 0,
                           port::MaxParallelism());
      thread_pool_ = absl::make_unique<thread::ThreadPool>(
          Env::Default(), ThreadOptions{}, "data_private_threadpool",
          threadpool_size_);
    }
    cancellation_manager_ = absl::make_unique<CancellationManager>();
  }

  ~Iterator() override { cancellation_manager_->StartCancel(); }

  Status Initialize(IteratorContext* ctx) override {
    return dataset()->input_->MakeIterator(IteratorContext(CreateParams(ctx)),
                                           this, prefix(), &input_impl_);
  }

  Status GetNextInternal(IteratorContext* ctx, std::vector<Tensor>* out_tensors,
                         bool* end_of_sequence) override {
    if (dataset()->params_.autotune) {
      TF_RETURN_IF_ERROR(EnsureModelThreadStarted(ctx));
    }
    return input_impl_->GetNext(IteratorContext(CreateParams(ctx)), out_tensors,
                                end_of_sequence);
  }

 protected:
  std::shared_ptr<model::Node> CreateNode(
      IteratorContext* ctx, model::Node::Args args) const override {
    return model::MakeKnownRatioNode(std::move(args), /*ratio=*/1);
  }

  Status SaveInternal(SerializationContext* ctx,
                      IteratorStateWriter* writer) override {
    TF_RETURN_IF_ERROR(SaveInput(ctx, writer, input_impl_));
    return OkStatus();
  }

  Status RestoreInternal(IteratorContext* ctx,
                         IteratorStateReader* reader) override {
    TF_RETURN_IF_ERROR(
        RestoreInput(IteratorContext(CreateParams(ctx)), reader, input_impl_));
    return OkStatus();
  }

  TraceMeMetadata GetTraceMeMetadata() const override {
    tensorflow::data::TraceMeMetadata traceme_metadata =
        dataset()->traceme_metadata_;
    const int64_t mem_bw = port::GetMemoryBandwidthInfo().bw_used;
    if (mem_bw != INT64_MAX) {
      traceme_metadata.push_back(std::make_pair(
          kMemBandwidth,
          strings::Printf("%lld", static_cast<long long>(mem_bw))));
    }
    const auto memory_info = port::GetMemoryInfo();
    const auto memory_usage = memory_info.total - memory_info.free;
    traceme_metadata.push_back(std::make_pair(
        kRamUsage,
        strings::Printf("%lld out of %lld (%.2f%%)",
                        static_cast<long long>(memory_usage / 1.0e6),
                        static_cast<long long>(memory_info.total / 1.0e6),
                        static_cast<double>(memory_usage) /
                            static_cast<double>(memory_info.total))));
    if (model_node() != nullptr) {
      traceme_metadata.push_back(std::make_pair(
          kMaxBufferBytes,
          strings::Printf(
              "%lld", static_cast<long long>(
                          model_node()->TotalMaximumBufferedBytes() / 1.0e6))));
    }
    return traceme_metadata;
  }

 private:
  IteratorContext::Params CreateParams(IteratorContext* ctx) {
    IteratorContext::Params params(ctx);
    if (dataset()->params_.autotune) {
      params.model = model_;
    }
    if (dataset()->params_.private_threadpool_size >= 0) {
      params.runner = [pool = thread_pool_.get()](std::function<void()> c) {
        pool->Schedule(std::move(c));
      };
      params.runner_threadpool_size = threadpool_size_;
    }
    if (dataset()->params_.max_intra_op_parallelism >= 0) {
      params.runner =
          RunnerWithMaxParallelism(params.runner, max_intra_op_parallelism_);
    }
    params.options = &dataset()->options();
    return params;
  }

  Status EnsureModelThreadStarted(IteratorContext* ctx) {
    mutex_lock l(mu_);
    if (!model_thread_) {
      model_thread_ = ctx->StartThread("tf_data_model", [this]() {
        Status status =
            model_->OptimizeLoop(dataset()->params_.autotune_algorithm,
                                 dataset()->params_.autotune_cpu_budget,
                                 dataset()->params_.autotune_ram_budget,
                                 cancellation_manager_.get());
        if (!status.ok()) {
          LOG(WARNING) << "Optimization loop failed: " << status.ToString();
        }
      });
    }
    return OkStatus();
  }

  std::shared_ptr<model::Model> model_ = nullptr;
  // Controls cancellation of `model_thread_`. Must be ordered before
  // `model_thread_` so that `model_thread_` is destroyed first.
  std::unique_ptr<CancellationManager> cancellation_manager_;
  mutex mu_;
  std::unique_ptr<Thread> model_thread_ TF_GUARDED_BY(mu_);
  int64_t max_intra_op_parallelism_;
  int64_t threadpool_size_;
  std::unique_ptr<thread::ThreadPool> thread_pool_;

  // Must be ordered last as its execution may depend on other members.
  std::unique_ptr<IteratorBase> input_impl_;
};

RootDataset::RootDataset(const DatasetBase* input, const Params& params)
    : DatasetBase(DatasetContext({name_utils::OpName(kDatasetType),
                                  name_utils::OpName(kDatasetType)})),
      input_(input),
      params_(std::move(params)) {
  AddTraceMetadata(params_, &traceme_metadata_);
}

RootDataset::RootDataset(core::RefCountPtr<DatasetBase> input,
                         const Params& params)
    : DatasetBase(DatasetContext({name_utils::OpName(kDatasetType),
                                  name_utils::OpName(kDatasetType)})),
      params_(std::move(params)) {
  owned_input_ = std::move(input);
  input_ = owned_input_.get();
  AddTraceMetadata(params_, &traceme_metadata_);
}

RootDataset::~RootDataset() {}

std::unique_ptr<IteratorBase> RootDataset::MakeIteratorInternal(
    const string& prefix) const {
  return absl::make_unique<Iterator>(
      Iterator::Params{this, name_utils::IteratorPrefix(kDatasetType, prefix)});
}

const DataTypeVector& RootDataset::output_dtypes() const {
  return input_->output_dtypes();
}

const std::vector<PartialTensorShape>& RootDataset::output_shapes() const {
  return input_->output_shapes();
}

string RootDataset::DebugString() const {
  return name_utils::DatasetDebugString(kDatasetType);
}

int64_t RootDataset::CardinalityInternal() const {
  return input_->Cardinality();
}

int64_t RootDataset::CardinalityInternal(CardinalityOptions options) const {
  return input_->Cardinality(options);
}

Status RootDataset::Get(OpKernelContext* ctx, int64 index,
                        std::vector<Tensor>* out_tensors) const {
  std::vector<const DatasetBase*> inputs;
  TF_RETURN_IF_ERROR(this->InputDatasets(&inputs));
  return inputs[0]->Get(ctx, index, out_tensors);
}

Status RootDataset::InputDatasets(
    std::vector<const DatasetBase*>* inputs) const {
  inputs->push_back(input_);
  return OkStatus();
}

Status RootDataset::CheckExternalState() const {
  return input_->CheckExternalState();
}

Status RootDataset::AsGraphDefInternal(SerializationContext* ctx,
                                       DatasetGraphDefBuilder* b,
                                       Node** output) const {
  return errors::Unimplemented("RootDataset does not support serialization.");
}

#if !defined(IS_MOBILE_PLATFORM)
Status FinalizeDataset(OpKernelContext* ctx, const DatasetBase* input,
                       DatasetBase** output) {
  const Options& options = input->options();
  absl::flat_hash_set<tstring> optimizations_enabled;
  absl::flat_hash_set<tstring> optimizations_disabled;
  absl::flat_hash_set<tstring> optimizations_default;
  GetOptimizations(options, &optimizations_enabled, &optimizations_disabled,
                   &optimizations_default);
  // Disable `enable_gradient_descent` as it assumes presence of ModelDatasetOp.
  optimizations_disabled.insert("enable_gradient_descent");
  if (!port::JobName().empty()) {
    // Enable kInjectPrefetchEligibleOpt that does not modify the graph and is
    // used to check whether the `inject_prefetch` optimization would modify the
    // graph.
    optimizations_enabled.insert(kInjectPrefetchEligibleOpt);
  }

  auto experiments = GetExperiments();
  LogAndRecordExperiments(experiments);
  auto optimizations =
      SelectOptimizations(experiments, optimizations_enabled,
                          optimizations_disabled, optimizations_default);
  if (optimizations.empty()) {
    return RootDataset::FromOptions(input, output);
  }

  auto optimization_configs = CreateGraphRewriteConfigs(options);
  auto config_factory = [&optimizations, &optimization_configs]() {
    return CreateRewriterConfig(optimizations, optimization_configs);
  };
  core::RefCountPtr<DatasetBase> rewritten_output;
  Status s = RewriteDataset(ctx, input, std::move(config_factory),
                            /*record_fingerprint=*/true, &rewritten_output);

  *output = rewritten_output.get();
  bool rewritten = (*output != input);
  if (errors::IsDeadlineExceeded(s)) {
    // Ignore DeadlineExceeded as it implies that the attempted rewrite took too
    // long which should not prevent further computation.
    LOG(WARNING) << s.ToString();
  } else if (!s.ok()) {
    return s;
  }
  if (!rewritten) {
    return RootDataset::FromOptions(input, output);
  } else {
    return RootDataset::FromOptions(std::move(rewritten_output), output);
  }
  return OkStatus();
}

#else   // !IS_MOBILE_PLATFORM
Status FinalizeDataset(OpKernelContext* ctx, const DatasetBase* input,
                       DatasetBase** output) {
  return RootDataset::FromOptions(input, output);
}
#endif  // !IS_MOBILE_PLATFORM

}  // namespace data
}  // namespace tensorflow
<commit_msg>#tf-data Fix memory usage percentage.<commit_after>/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/data/root_dataset.h"

#include <functional>
#include <string>
#include <utility>

#include "tensorflow/core/data/dataset_utils.h"
#include "tensorflow/core/data/name_utils.h"
#include "tensorflow/core/data/rewrite_utils.h"
#include "tensorflow/core/framework/model.pb.h"
#include "tensorflow/core/platform/errors.h"
#include "tensorflow/core/platform/host_info.h"
#include "tensorflow/core/platform/refcount.h"
#include "tensorflow/core/platform/stringprintf.h"

namespace tensorflow {
namespace data {
namespace {

constexpr char kDatasetType[] = "Root";

constexpr char kAlgorithm[] = "algorithm";
constexpr char kCpuBudget[] = "cpu_budget";
constexpr char kExperiments[] = "experiments";
constexpr char kInjectPrefetchEligibleOpt[] = "inject_prefetch_eligible";
constexpr char kIntraOpParallelism[] = "intra_op_parallelism";
constexpr char kMemBandwidth[] = "mem_bw_used_megabytes_per_sec";
constexpr char kPrivateThreadpoolSize[] = "threadpool_size";
constexpr char kRamBudget[] = "ram_budget_megabytes";
constexpr char kRamUsage[] = "ram_usage_megabytes";
constexpr char kMaxBufferBytes[] = "max_buffered_megabytes";

// If value `x` matches `y`, returns default value `z`. Otherwise, return `x`.
inline int64_t value_or_default(int64_t x, int64_t y, int64_t z) {
  return x == y ? z : x;
}

void SetRootDatasetParams(const Options& options, RootDataset::Params* params) {
  if (ShouldConfigureMaxIntraOpParallelism(options)) {
    params->max_intra_op_parallelism =
        options.threading_options().max_intra_op_parallelism();
  }
  if (ShouldUsePrivateThreadPool(options)) {
    params->private_threadpool_size =
        options.threading_options().private_threadpool_size();
  }
  params->autotune = ShouldUseAutotuning(options);
  if (params->autotune) {
    params->autotune_algorithm =
        options.autotune_options().optional_autotune_algorithm_case() ==
                AutotuneOptions::kAutotuneAlgorithm
            ? options.autotune_options().autotune_algorithm()
            : model::AutotuneAlgorithm::DEFAULT;
    params->autotune_cpu_budget = value_or_default(
        options.autotune_options().cpu_budget(), 0, GetCpuBudget());
    params->autotune_ram_budget =
        value_or_default(options.autotune_options().ram_budget(), 0,
                         model::kRamBudgetShare * port::AvailableRam());
  }
}

void AddTraceMetadata(const RootDataset::Params& params,
                      TraceMeMetadata* trace_metadata) {
  if (params.autotune) {
    trace_metadata->push_back(std::make_pair(
        kAlgorithm, model::AutotuneAlgorithm_Name(params.autotune_algorithm)));
    trace_metadata->push_back(std::make_pair(
        kCpuBudget, strings::Printf("%lld", static_cast<long long>(
                                                params.autotune_cpu_budget))));
    trace_metadata->push_back(std::make_pair(
        kRamBudget,
        strings::Printf("%lld", static_cast<long long>(
                                    params.autotune_ram_budget / 1.0e6))));
  }
  if (params.max_intra_op_parallelism >= 0) {
    trace_metadata->push_back(std::make_pair(
        kIntraOpParallelism,
        strings::Printf("%lld", static_cast<long long>(value_or_default(
                                    params.max_intra_op_parallelism, 0,
                                    port::MaxParallelism())))));
  }
  if (params.private_threadpool_size >= 0) {
    trace_metadata->push_back(std::make_pair(
        kPrivateThreadpoolSize,
        strings::Printf("%lld", static_cast<long long>(value_or_default(
                                    params.private_threadpool_size, 0,
                                    port::MaxParallelism())))));
  }
  auto experiments = GetExperiments();
  if (!experiments.empty()) {
    trace_metadata->push_back(
        std::make_pair(kExperiments, absl::StrJoin(experiments, " ")));
  }
}
}  // namespace

// static
Status RootDataset::FromOptions(const DatasetBase* input,
                                DatasetBase** output) {
  Params params;
  SetRootDatasetParams(input->options(), &params);
  *output = new RootDataset(input, params);
  (*output)->Initialize(/*metadata=*/{});
  return OkStatus();
}

Status RootDataset::FromOptions(core::RefCountPtr<DatasetBase> input,
                                DatasetBase** output) {
  Params params;
  SetRootDatasetParams(input->options(), &params);
  *output = new RootDataset(std::move(input), params);
  (*output)->Initialize(/*metadata=*/{});
  return OkStatus();
}

class RootDataset::Iterator : public DatasetIterator<RootDataset> {
 public:
  explicit Iterator(const Params& params)
      : DatasetIterator<RootDataset>(params) {
    if (dataset()->params_.autotune) {
      model_ = std::make_shared<model::Model>();
    }
    if (dataset()->params_.max_intra_op_parallelism >= 0) {
      max_intra_op_parallelism_ =
          value_or_default(dataset()->params_.max_intra_op_parallelism, 0,
                           port::MaxParallelism());
    }
    if (dataset()->params_.private_threadpool_size >= 0) {
      threadpool_size_ =
          value_or_default(dataset()->params_.private_threadpool_size, 0,
                           port::MaxParallelism());
      thread_pool_ = absl::make_unique<thread::ThreadPool>(
          Env::Default(), ThreadOptions{}, "data_private_threadpool",
          threadpool_size_);
    }
    cancellation_manager_ = absl::make_unique<CancellationManager>();
  }

  ~Iterator() override { cancellation_manager_->StartCancel(); }

  Status Initialize(IteratorContext* ctx) override {
    return dataset()->input_->MakeIterator(IteratorContext(CreateParams(ctx)),
                                           this, prefix(), &input_impl_);
  }

  Status GetNextInternal(IteratorContext* ctx, std::vector<Tensor>* out_tensors,
                         bool* end_of_sequence) override {
    if (dataset()->params_.autotune) {
      TF_RETURN_IF_ERROR(EnsureModelThreadStarted(ctx));
    }
    return input_impl_->GetNext(IteratorContext(CreateParams(ctx)), out_tensors,
                                end_of_sequence);
  }

 protected:
  std::shared_ptr<model::Node> CreateNode(
      IteratorContext* ctx, model::Node::Args args) const override {
    return model::MakeKnownRatioNode(std::move(args), /*ratio=*/1);
  }

  Status SaveInternal(SerializationContext* ctx,
                      IteratorStateWriter* writer) override {
    TF_RETURN_IF_ERROR(SaveInput(ctx, writer, input_impl_));
    return OkStatus();
  }

  Status RestoreInternal(IteratorContext* ctx,
                         IteratorStateReader* reader) override {
    TF_RETURN_IF_ERROR(
        RestoreInput(IteratorContext(CreateParams(ctx)), reader, input_impl_));
    return OkStatus();
  }

  TraceMeMetadata GetTraceMeMetadata() const override {
    tensorflow::data::TraceMeMetadata traceme_metadata =
        dataset()->traceme_metadata_;
    const int64_t mem_bw = port::GetMemoryBandwidthInfo().bw_used;
    if (mem_bw != INT64_MAX) {
      traceme_metadata.push_back(std::make_pair(
          kMemBandwidth,
          strings::Printf("%lld", static_cast<long long>(mem_bw))));
    }
    const auto memory_info = port::GetMemoryInfo();
    const auto memory_usage = memory_info.total - memory_info.free;
    traceme_metadata.push_back(std::make_pair(
        kRamUsage,
        strings::Printf("%lld out of %lld (%.2f%%)",
                        static_cast<long long>(memory_usage / 1.0e6),
                        static_cast<long long>(memory_info.total / 1.0e6),
                        static_cast<double>(100 * memory_usage) /
                            static_cast<double>(memory_info.total))));
    if (model_node() != nullptr) {
      traceme_metadata.push_back(std::make_pair(
          kMaxBufferBytes,
          strings::Printf(
              "%lld", static_cast<long long>(
                          model_node()->TotalMaximumBufferedBytes() / 1.0e6))));
    }
    return traceme_metadata;
  }

 private:
  IteratorContext::Params CreateParams(IteratorContext* ctx) {
    IteratorContext::Params params(ctx);
    if (dataset()->params_.autotune) {
      params.model = model_;
    }
    if (dataset()->params_.private_threadpool_size >= 0) {
      params.runner = [pool = thread_pool_.get()](std::function<void()> c) {
        pool->Schedule(std::move(c));
      };
      params.runner_threadpool_size = threadpool_size_;
    }
    if (dataset()->params_.max_intra_op_parallelism >= 0) {
      params.runner =
          RunnerWithMaxParallelism(params.runner, max_intra_op_parallelism_);
    }
    params.options = &dataset()->options();
    return params;
  }

  Status EnsureModelThreadStarted(IteratorContext* ctx) {
    mutex_lock l(mu_);
    if (!model_thread_) {
      model_thread_ = ctx->StartThread("tf_data_model", [this]() {
        Status status =
            model_->OptimizeLoop(dataset()->params_.autotune_algorithm,
                                 dataset()->params_.autotune_cpu_budget,
                                 dataset()->params_.autotune_ram_budget,
                                 cancellation_manager_.get());
        if (!status.ok()) {
          LOG(WARNING) << "Optimization loop failed: " << status.ToString();
        }
      });
    }
    return OkStatus();
  }

  std::shared_ptr<model::Model> model_ = nullptr;
  // Controls cancellation of `model_thread_`. Must be ordered before
  // `model_thread_` so that `model_thread_` is destroyed first.
  std::unique_ptr<CancellationManager> cancellation_manager_;
  mutex mu_;
  std::unique_ptr<Thread> model_thread_ TF_GUARDED_BY(mu_);
  int64_t max_intra_op_parallelism_;
  int64_t threadpool_size_;
  std::unique_ptr<thread::ThreadPool> thread_pool_;

  // Must be ordered last as its execution may depend on other members.
  std::unique_ptr<IteratorBase> input_impl_;
};

RootDataset::RootDataset(const DatasetBase* input, const Params& params)
    : DatasetBase(DatasetContext({name_utils::OpName(kDatasetType),
                                  name_utils::OpName(kDatasetType)})),
      input_(input),
      params_(std::move(params)) {
  AddTraceMetadata(params_, &traceme_metadata_);
}

RootDataset::RootDataset(core::RefCountPtr<DatasetBase> input,
                         const Params& params)
    : DatasetBase(DatasetContext({name_utils::OpName(kDatasetType),
                                  name_utils::OpName(kDatasetType)})),
      params_(std::move(params)) {
  owned_input_ = std::move(input);
  input_ = owned_input_.get();
  AddTraceMetadata(params_, &traceme_metadata_);
}

RootDataset::~RootDataset() {}

std::unique_ptr<IteratorBase> RootDataset::MakeIteratorInternal(
    const string& prefix) const {
  return absl::make_unique<Iterator>(
      Iterator::Params{this, name_utils::IteratorPrefix(kDatasetType, prefix)});
}

const DataTypeVector& RootDataset::output_dtypes() const {
  return input_->output_dtypes();
}

const std::vector<PartialTensorShape>& RootDataset::output_shapes() const {
  return input_->output_shapes();
}

string RootDataset::DebugString() const {
  return name_utils::DatasetDebugString(kDatasetType);
}

int64_t RootDataset::CardinalityInternal() const {
  return input_->Cardinality();
}

int64_t RootDataset::CardinalityInternal(CardinalityOptions options) const {
  return input_->Cardinality(options);
}

Status RootDataset::Get(OpKernelContext* ctx, int64 index,
                        std::vector<Tensor>* out_tensors) const {
  std::vector<const DatasetBase*> inputs;
  TF_RETURN_IF_ERROR(this->InputDatasets(&inputs));
  return inputs[0]->Get(ctx, index, out_tensors);
}

Status RootDataset::InputDatasets(
    std::vector<const DatasetBase*>* inputs) const {
  inputs->push_back(input_);
  return OkStatus();
}

Status RootDataset::CheckExternalState() const {
  return input_->CheckExternalState();
}

Status RootDataset::AsGraphDefInternal(SerializationContext* ctx,
                                       DatasetGraphDefBuilder* b,
                                       Node** output) const {
  return errors::Unimplemented("RootDataset does not support serialization.");
}

#if !defined(IS_MOBILE_PLATFORM)
Status FinalizeDataset(OpKernelContext* ctx, const DatasetBase* input,
                       DatasetBase** output) {
  const Options& options = input->options();
  absl::flat_hash_set<tstring> optimizations_enabled;
  absl::flat_hash_set<tstring> optimizations_disabled;
  absl::flat_hash_set<tstring> optimizations_default;
  GetOptimizations(options, &optimizations_enabled, &optimizations_disabled,
                   &optimizations_default);
  // Disable `enable_gradient_descent` as it assumes presence of ModelDatasetOp.
  optimizations_disabled.insert("enable_gradient_descent");
  if (!port::JobName().empty()) {
    // Enable kInjectPrefetchEligibleOpt that does not modify the graph and is
    // used to check whether the `inject_prefetch` optimization would modify the
    // graph.
    optimizations_enabled.insert(kInjectPrefetchEligibleOpt);
  }

  auto experiments = GetExperiments();
  LogAndRecordExperiments(experiments);
  auto optimizations =
      SelectOptimizations(experiments, optimizations_enabled,
                          optimizations_disabled, optimizations_default);
  if (optimizations.empty()) {
    return RootDataset::FromOptions(input, output);
  }

  auto optimization_configs = CreateGraphRewriteConfigs(options);
  auto config_factory = [&optimizations, &optimization_configs]() {
    return CreateRewriterConfig(optimizations, optimization_configs);
  };
  core::RefCountPtr<DatasetBase> rewritten_output;
  Status s = RewriteDataset(ctx, input, std::move(config_factory),
                            /*record_fingerprint=*/true, &rewritten_output);

  *output = rewritten_output.get();
  bool rewritten = (*output != input);
  if (errors::IsDeadlineExceeded(s)) {
    // Ignore DeadlineExceeded as it implies that the attempted rewrite took too
    // long which should not prevent further computation.
    LOG(WARNING) << s.ToString();
  } else if (!s.ok()) {
    return s;
  }
  if (!rewritten) {
    return RootDataset::FromOptions(input, output);
  } else {
    return RootDataset::FromOptions(std::move(rewritten_output), output);
  }
  return OkStatus();
}

#else   // !IS_MOBILE_PLATFORM
Status FinalizeDataset(OpKernelContext* ctx, const DatasetBase* input,
                       DatasetBase** output) {
  return RootDataset::FromOptions(input, output);
}
#endif  // !IS_MOBILE_PLATFORM

}  // namespace data
}  // namespace tensorflow
<|endoftext|>


<commit_before>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/grappler/op_types.h"

namespace tensorflow {
namespace grappler {

bool IsAddN(const NodeDef& node) {
  const auto op = node.op();
  return op == "AddN";
}

bool IsConcat(const NodeDef& node) {
  const auto op = node.op();
  return op == "Concat" || op == "ConcatV2";
}

bool IsConstant(const NodeDef& node) {
  const auto op = node.op();
  return op == "Const";
}

bool IsDequeueOp(const NodeDef& node) {
  const auto& op = node.op();
  return op == "QueueDequeueManyV2" || op == "QueueDequeueMany" ||
         op == "QueueDequeueV2" || op == "QueueDequeue" ||
         op == "QueueDequeueUpToV2" || op == "QueueDequeueUpTo";
}

bool IsIdentity(const NodeDef& node) {
  const auto& op = node.op();
  return op == "Identity" || op == "RefIdentity";
}

bool IsMerge(const NodeDef& node) {
  const auto op = node.op();
  return op == "Merge";
}

bool IsNoOp(const NodeDef& node) {
  const auto op = node.op();
  return op == "NoOp";
}

bool IsNextIteration(const NodeDef& node) {
  const auto& op = node.op();
  return op == "NextIteration" || op == "RefNextIteration";
}

bool IsPlaceholder(const NodeDef& node) {
  const auto op = node.op();
  return op == "Placeholder" || op == "PlaceholderV2" ||
         op == "PlaceholderWithDefault";
}

bool IsRecv(const NodeDef& node) {
  const auto op = node.op();
  return op == "_Recv";
}

bool IsReduction(const NodeDef& node) {
  const auto& op = node.op();
  return op == "Sum" || op == "Prod" || op == "Min" || op == "Max" ||
         op == "Mean" || op == "Any" || op == "All";
}

bool IsReshape(const NodeDef& node) { return (node.op() == "Reshape"); }

bool IsSend(const NodeDef& node) {
  const auto op = node.op();
  return op == "_Send";
}

bool IsStopGradient(const NodeDef& node) {
  const auto& op = node.op();
  return op == "StopGradient" || op == "PreventGradient";
}

bool IsSwitch(const NodeDef& node) {
  const auto& op = node.op();
  return op == "Switch";
}

bool IsTranspose(const NodeDef& node) {
  const auto op = node.op();
  return op == "Transpose";
}

bool IsVariable(const NodeDef& node) {
  const auto op = node.op();
  return op == "Variable" || op == "VariableV2" || op == "AutoReloadVariable" ||
         op == "VarHandleOp" || op == "TemporaryVariable";
}

}  // end namespace grappler
}  // end namespace tensorflow
<commit_msg>Added support for RefMerge and RefSwitch to grappler<commit_after>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/grappler/op_types.h"

namespace tensorflow {
namespace grappler {

bool IsAddN(const NodeDef& node) {
  const auto op = node.op();
  return op == "AddN";
}

bool IsConcat(const NodeDef& node) {
  const auto op = node.op();
  return op == "Concat" || op == "ConcatV2";
}

bool IsConstant(const NodeDef& node) {
  const auto op = node.op();
  return op == "Const";
}

bool IsDequeueOp(const NodeDef& node) {
  const auto& op = node.op();
  return op == "QueueDequeueManyV2" || op == "QueueDequeueMany" ||
         op == "QueueDequeueV2" || op == "QueueDequeue" ||
         op == "QueueDequeueUpToV2" || op == "QueueDequeueUpTo";
}

bool IsIdentity(const NodeDef& node) {
  const auto& op = node.op();
  return op == "Identity" || op == "RefIdentity";
}

bool IsMerge(const NodeDef& node) {
  const auto op = node.op();
  return op == "Merge" || op == "RefMerge";
}

bool IsNoOp(const NodeDef& node) {
  const auto op = node.op();
  return op == "NoOp";
}

bool IsNextIteration(const NodeDef& node) {
  const auto& op = node.op();
  return op == "NextIteration" || op == "RefNextIteration";
}

bool IsPlaceholder(const NodeDef& node) {
  const auto op = node.op();
  return op == "Placeholder" || op == "PlaceholderV2" ||
         op == "PlaceholderWithDefault";
}

bool IsRecv(const NodeDef& node) {
  const auto op = node.op();
  return op == "_Recv";
}

bool IsReduction(const NodeDef& node) {
  const auto& op = node.op();
  return op == "Sum" || op == "Prod" || op == "Min" || op == "Max" ||
         op == "Mean" || op == "Any" || op == "All";
}

bool IsReshape(const NodeDef& node) { return (node.op() == "Reshape"); }

bool IsSend(const NodeDef& node) {
  const auto op = node.op();
  return op == "_Send";
}

bool IsStopGradient(const NodeDef& node) {
  const auto& op = node.op();
  return op == "StopGradient" || op == "PreventGradient";
}

bool IsSwitch(const NodeDef& node) {
  const auto& op = node.op();
  return op == "Switch" || op == "RefSwitch";
}

bool IsTranspose(const NodeDef& node) {
  const auto op = node.op();
  return op == "Transpose";
}

bool IsVariable(const NodeDef& node) {
  const auto op = node.op();
  return op == "Variable" || op == "VariableV2" || op == "AutoReloadVariable" ||
         op == "VarHandleOp" || op == "TemporaryVariable";
}

}  // end namespace grappler
}  // end namespace tensorflow
<|endoftext|>


<commit_before><commit_msg>cf-gltf bone weights renormalization (probably not always necessary!)<commit_after><|endoftext|>