<commit_before>#include <sbml/SBMLNamespaces.h>
#include <sstream>
#include <sbml/common/common.h>
#include <iostream>

/** @cond doxygenIgnored */

using namespace std;

/** @endcond */

LIBSBML_CPP_NAMESPACE_BEGIN

#ifdef __cplusplus

/** @cond doxygenLibsbmlInternal */
void
SBMLNamespaces::initSBMLNamespace()
{
  mNamespaces = new XMLNamespaces();

  switch (mLevel)
  {
  case 1:
  default:
    switch (mVersion)
    {
    case 1:
    default:
      mNamespaces->add(SBML_XMLNS_L1V1);
      break;
    }
    break;
  }

  if (mNamespaces->getLength() == 0)
  {
    mLevel = SBML_INT_MAX;
    mVersion = SBML_INT_MAX;
    delete mNamespaces;
    mNamespaces = NULL;
  }
}
/** @endcond */

SBMLNamespaces::SBMLNamespaces(unsigned int level, unsigned int version)
 : mLevel(level)
  ,mVersion(version)
{
  initSBMLNamespace();
}

SBMLNamespaces::~SBMLNamespaces()
{
  if (mNamespaces != NULL)
    delete mNamespaces;
}

/*
 * Copy constructor; creates a copy of a SBMLNamespaces.
 */
SBMLNamespaces::SBMLNamespaces(const SBMLNamespaces& orig)
 : mLevel(orig.mLevel)
 , mVersion(orig.mVersion)
 , mNamespaces(NULL)
{
  if(orig.mNamespaces != NULL)
    this->mNamespaces =
          new XMLNamespaces(*const_cast<SBMLNamespaces&>(orig).mNamespaces);
}

const List *
SBMLNamespaces::getSupportedNamespaces()
{
  List *result = new List();
  result->add(new SBMLNamespaces(1,1));
  return result;
}

void
SBMLNamespaces::freeSBMLNamespaces(List * supportedNS)
{
  if (supportedNS == NULL) return;
  for (unsigned int i = 0; i < supportedNS->getSize(); i++)
  {
    delete (SBMLNamespaces*)supportedNS->get(i);
  }
  delete supportedNS;
}

/*
 * Assignment operator for SBMLNamespaces.
 */
SBMLNamespaces&
SBMLNamespaces::operator=(const SBMLNamespaces& rhs)
{
  if (&rhs != this)
  {
    mLevel   = rhs.mLevel;
    mVersion = rhs.mVersion;
    delete this->mNamespaces;
    if(rhs.mNamespaces != NULL)
      this->mNamespaces =
            new XMLNamespaces(*const_cast<SBMLNamespaces&>(rhs).mNamespaces);
    else
      this->mNamespaces = NULL;
  }

  return *this;
}

/*
 * Creates and returns a deep copy of this SBMLNamespaces.
 */
SBMLNamespaces *
SBMLNamespaces::clone () const
{
  return new SBMLNamespaces(*this);
}

std::string
SBMLNamespaces::getSBMLNamespaceURI(unsigned int level,
                                 unsigned int version)
{
  std::string uri = "";
  switch (level)
  {
  case 1:
  default:
    switch (version)
    {
    case 1:
    default:
      uri = SBML_XMLNS_L1V1;
      break;
    }
    break;
  }
  return uri;
}

std::string
SBMLNamespaces::getURI() const
{
  return getSBMLNamespaceURI(mLevel,mVersion);
}

unsigned int
SBMLNamespaces::getLevel()
{
  return mLevel;
}

unsigned int
SBMLNamespaces::getLevel() const
{
  return mLevel;
}

unsigned int
SBMLNamespaces::getVersion()
{
  return mVersion;
}

unsigned int
SBMLNamespaces::getVersion() const
{
  return mVersion;
}

XMLNamespaces *
SBMLNamespaces::getNamespaces()
{
  return mNamespaces;
}

const XMLNamespaces *
SBMLNamespaces::getNamespaces() const
{
  return mNamespaces;
}

int
SBMLNamespaces::addNamespaces(const XMLNamespaces * xmlns)
{
  int success = LIBSBML_OPERATION_SUCCESS;

  if (xmlns == NULL)
    return LIBSBML_INVALID_OBJECT;

  if (!mNamespaces)
  {
    initSBMLNamespace();
  }

  /* check whether the namespace already exists
   * add if it does not
   */
  for (int i = 0; i < xmlns->getLength(); i++)
  {
    if (mNamespaces != NULL && !(mNamespaces->hasNS(xmlns->getURI(i), xmlns->getPrefix(i))))
    {
      success = mNamespaces->add(xmlns->getURI(i), xmlns->getPrefix(i));
    }
  }

  return success;
}

int
SBMLNamespaces::addNamespace(const std::string &uri, const std::string &prefix)
{
  if (!mNamespaces)
  {
    initSBMLNamespace();
  }

  return mNamespaces != NULL ? mNamespaces->add(uri, prefix) : LIBSBML_INVALID_OBJECT;
}

int
SBMLNamespaces::removeNamespace(const std::string &uri)
{
  if (!mNamespaces)
  {
    initSBMLNamespace();
  }

  return mNamespaces != NULL ? mNamespaces->remove(mNamespaces->getIndex(uri)) : LIBSBML_INVALID_OBJECT;
}

/*
 * Predicate returning @c true if the given
 * URL is one of SBML_Lang XML namespaces.
 */
bool
SBMLNamespaces::isSBMLNamespace(const std::string& uri)
{
  if (uri == SBML_XMLNS_L1V1)   return true;

  return false;
}

bool
SBMLNamespaces::isValidCombination()
{
  bool valid = true;
  bool sbmlDeclared = false;
  std::string declaredURI("");
  unsigned int version = getVersion();
  XMLNamespaces *xmlns = getNamespaces();

  if (xmlns != NULL)
  {
    int numNS = 0;

    if (xmlns->hasURI(SBML_XMLNS_L1V1))
    {
      ++numNS;
      declaredURI.assign(SBML_XMLNS_L1V1);
    }

    // checks if the SBML_Lang Namespace is explicitly defined.
    for (int i=0; i < xmlns->getLength(); i++)
    {
      if (!declaredURI.empty() &&
                      xmlns->getURI(i) == declaredURI)
      {
        sbmlDeclared = true;
        break;
      }
    }
  }

  switch (getLevel())
  {
    case 1:
     switch (version)
      {
        case 1:
          // the namespaces contains the sbml namespaces
          // check it is the correct ns for the level/version
          if (sbmlDeclared)
          {
            if (declaredURI != string(SBML_XMLNS_L1V1))
            {
              valid = false;
            }
          }
          break;
        default:
          valid = false;
          break;
        }
      break;
    default:
      valid = false;
      break;
  }

  return valid;
}

/** @cond doxygenLibsbmlInternal */
void
SBMLNamespaces::setLevel(unsigned int level)
{
  mLevel = level;
}

void
SBMLNamespaces::setVersion(unsigned int version)
{
  mVersion = version;
}

void
SBMLNamespaces::setNamespaces(XMLNamespaces * xmlns)
{
  delete mNamespaces;
  if (xmlns != NULL)
    mNamespaces = xmlns->clone();
  else
    mNamespaces = NULL;
}
/** @endcond */

#endif /* __cplusplus */

/** @cond doxygenIgnored */

LIBSBML_EXTERN
SBMLNamespaces_t *
SBMLNamespaces_create(unsigned int level, unsigned int version)
{
  return new SBMLNamespaces(level, version);
}

LIBSBML_EXTERN
void
SBMLNamespaces_free(SBMLNamespaces_t* ns)
{
  if (ns == NULL) return;
  delete static_cast<SBMLNamespaces*>(ns);
}

LIBSBML_EXTERN
unsigned int
SBMLNamespaces_getLevel(SBMLNamespaces_t *sbmlns)
{
  return (sbmlns != NULL) ? sbmlns->getLevel() : SBML_INT_MAX;
}

LIBSBML_EXTERN
unsigned int
SBMLNamespaces_getVersion(SBMLNamespaces_t *sbmlns)
{
  return (sbmlns != NULL) ? sbmlns->getVersion() : SBML_INT_MAX;
}

LIBSBML_EXTERN
XMLNamespaces_t *
SBMLNamespaces_getNamespaces(SBMLNamespaces_t *sbmlns)
{
  return (sbmlns != NULL) ? sbmlns->getNamespaces() : NULL;
}

LIBSBML_EXTERN
char *
SBMLNamespaces_getSBMLNamespaceURI(unsigned int level, unsigned int version)
{
  return safe_strdup(SBMLNamespaces::getSBMLNamespaceURI(level, version).c_str());
}

LIBSBML_EXTERN
int
SBMLNamespaces_addNamespaces(SBMLNamespaces_t *sbmlns,
                             const XMLNamespaces_t * xmlns)
{
  if (sbmlns != NULL)
    return sbmlns->addNamespaces(xmlns);
  else
    return LIBSBML_INVALID_OBJECT;
}

LIBSBML_EXTERN
SBMLNamespaces_t **
SBMLNamespaces_getSupportedNamespaces(int *length)
{
  if (length == NULL) return NULL;
   const List* supported = SBMLNamespaces::getSupportedNamespaces();

   *length = (int) supported->getSize();
  SBMLNamespaces_t ** result = (SBMLNamespaces_t**)malloc(sizeof(SBMLNamespaces_t*)*((unsigned long)*length));
  memset(result, 0, sizeof(SBMLNamespaces_t*)*((unsigned long)*length));
  for (int i = 0; i < *length; i++)
  {
    result[i] = ((SBMLNamespaces*)supported->get((unsigned int)i))->clone();
  }
  SBMLNamespaces::freeSBMLNamespaces(const_cast<List*>(supported));
  return result;
}
/** @endcond */

LIBSBML_CPP_NAMESPACE_END

<commit_msg>use level/version/ns from description<commit_after>#include <sbml/SBMLNamespaces.h>
#include <sstream>
#include <sbml/common/common.h>
#include <iostream>

/** @cond doxygenIgnored */

using namespace std;

/** @endcond */

LIBSBML_CPP_NAMESPACE_BEGIN

#ifdef __cplusplus

/** @cond doxygenLibsbmlInternal */
void
SBMLNamespaces::initSBMLNamespace()
{
  mNamespaces = new XMLNamespaces();

  switch (mLevel)
  {
  case <SPEC_LEVEL>:
  default:
    switch (mVersion)
    {
    case <SPEC_VERSION>:
    default:
      mNamespaces->add(SBML_XMLNS_L<SPEC_LEVEL>V<SPEC_VERSION>);
      break;
    }
    break;
  }

  if (mNamespaces->getLength() == 0)
  {
    mLevel = SBML_INT_MAX;
    mVersion = SBML_INT_MAX;
    delete mNamespaces;
    mNamespaces = NULL;
  }
}
/** @endcond */

SBMLNamespaces::SBMLNamespaces(unsigned int level, unsigned int version)
 : mLevel(level)
  ,mVersion(version)
{
  initSBMLNamespace();
}

SBMLNamespaces::~SBMLNamespaces()
{
  if (mNamespaces != NULL)
    delete mNamespaces;
}

/*
 * Copy constructor; creates a copy of a SBMLNamespaces.
 */
SBMLNamespaces::SBMLNamespaces(const SBMLNamespaces& orig)
 : mLevel(orig.mLevel)
 , mVersion(orig.mVersion)
 , mNamespaces(NULL)
{
  if(orig.mNamespaces != NULL)
    this->mNamespaces =
          new XMLNamespaces(*const_cast<SBMLNamespaces&>(orig).mNamespaces);
}

const List *
SBMLNamespaces::getSupportedNamespaces()
{
  List *result = new List();
  result->add(new SBMLNamespaces(<SPEC_LEVEL>,<SPEC_VERSION>));
  return result;
}

void
SBMLNamespaces::freeSBMLNamespaces(List * supportedNS)
{
  if (supportedNS == NULL) return;
  for (unsigned int i = 0; i < supportedNS->getSize(); i++)
  {
    delete (SBMLNamespaces*)supportedNS->get(i);
  }
  delete supportedNS;
}

/*
 * Assignment operator for SBMLNamespaces.
 */
SBMLNamespaces&
SBMLNamespaces::operator=(const SBMLNamespaces& rhs)
{
  if (&rhs != this)
  {
    mLevel   = rhs.mLevel;
    mVersion = rhs.mVersion;
    delete this->mNamespaces;
    if(rhs.mNamespaces != NULL)
      this->mNamespaces =
            new XMLNamespaces(*const_cast<SBMLNamespaces&>(rhs).mNamespaces);
    else
      this->mNamespaces = NULL;
  }

  return *this;
}

/*
 * Creates and returns a deep copy of this SBMLNamespaces.
 */
SBMLNamespaces *
SBMLNamespaces::clone () const
{
  return new SBMLNamespaces(*this);
}

std::string
SBMLNamespaces::getSBMLNamespaceURI(unsigned int level,
                                 unsigned int version)
{
  std::string uri = "";
  switch (level)
  {
  case <SPEC_LEVEL>:
  default:
    switch (version)
    {
    case <SPEC_VERSION>:
    default:
      uri = SBML_XMLNS_L<SPEC_LEVEL>V<SPEC_VERSION>;
      break;
    }
    break;
  }
  return uri;
}

std::string
SBMLNamespaces::getURI() const
{
  return getSBMLNamespaceURI(mLevel,mVersion);
}

unsigned int
SBMLNamespaces::getLevel()
{
  return mLevel;
}

unsigned int
SBMLNamespaces::getLevel() const
{
  return mLevel;
}

unsigned int
SBMLNamespaces::getVersion()
{
  return mVersion;
}

unsigned int
SBMLNamespaces::getVersion() const
{
  return mVersion;
}

XMLNamespaces *
SBMLNamespaces::getNamespaces()
{
  return mNamespaces;
}

const XMLNamespaces *
SBMLNamespaces::getNamespaces() const
{
  return mNamespaces;
}

int
SBMLNamespaces::addNamespaces(const XMLNamespaces * xmlns)
{
  int success = LIBSBML_OPERATION_SUCCESS;

  if (xmlns == NULL)
    return LIBSBML_INVALID_OBJECT;

  if (!mNamespaces)
  {
    initSBMLNamespace();
  }

  /* check whether the namespace already exists
   * add if it does not
   */
  for (int i = 0; i < xmlns->getLength(); i++)
  {
    if (mNamespaces != NULL && !(mNamespaces->hasNS(xmlns->getURI(i), xmlns->getPrefix(i))))
    {
      success = mNamespaces->add(xmlns->getURI(i), xmlns->getPrefix(i));
    }
  }

  return success;
}

int
SBMLNamespaces::addNamespace(const std::string &uri, const std::string &prefix)
{
  if (!mNamespaces)
  {
    initSBMLNamespace();
  }

  return mNamespaces != NULL ? mNamespaces->add(uri, prefix) : LIBSBML_INVALID_OBJECT;
}

int
SBMLNamespaces::removeNamespace(const std::string &uri)
{
  if (!mNamespaces)
  {
    initSBMLNamespace();
  }

  return mNamespaces != NULL ? mNamespaces->remove(mNamespaces->getIndex(uri)) : LIBSBML_INVALID_OBJECT;
}

/*
 * Predicate returning @c true if the given
 * URL is one of SBML_Lang XML namespaces.
 */
bool
SBMLNamespaces::isSBMLNamespace(const std::string& uri)
{
  if (uri == SBML_XMLNS_L<SPEC_LEVEL>V<SPEC_VERSION>)   return true;

  return false;
}

bool
SBMLNamespaces::isValidCombination()
{
  bool valid = true;
  bool sbmlDeclared = false;
  std::string declaredURI("");
  unsigned int version = getVersion();
  XMLNamespaces *xmlns = getNamespaces();

  if (xmlns != NULL)
  {
    int numNS = 0;

    if (xmlns->hasURI(SBML_XMLNS_L<SPEC_LEVEL>V<SPEC_VERSION>))
    {
      ++numNS;
      declaredURI.assign(SBML_XMLNS_L<SPEC_LEVEL>V<SPEC_VERSION>);
    }

    // checks if the SBML_Lang Namespace is explicitly defined.
    for (int i=0; i < xmlns->getLength(); i++)
    {
      if (!declaredURI.empty() &&
                      xmlns->getURI(i) == declaredURI)
      {
        sbmlDeclared = true;
        break;
      }
    }
  }

  switch (getLevel())
  {
    case <SPEC_LEVEL>:
     switch (version)
      {
        case <SPEC_VERSION>:
          // the namespaces contains the sbml namespaces
          // check it is the correct ns for the level/version
          if (sbmlDeclared)
          {
            if (declaredURI != string(SBML_XMLNS_L<SPEC_LEVEL>V<SPEC_VERSION>))
            {
              valid = false;
            }
          }
          break;
        default:
          valid = false;
          break;
        }
      break;
    default:
      valid = false;
      break;
  }

  return valid;
}

/** @cond doxygenLibsbmlInternal */
void
SBMLNamespaces::setLevel(unsigned int level)
{
  mLevel = level;
}

void
SBMLNamespaces::setVersion(unsigned int version)
{
  mVersion = version;
}

void
SBMLNamespaces::setNamespaces(XMLNamespaces * xmlns)
{
  delete mNamespaces;
  if (xmlns != NULL)
    mNamespaces = xmlns->clone();
  else
    mNamespaces = NULL;
}
/** @endcond */

#endif /* __cplusplus */

/** @cond doxygenIgnored */

LIBSBML_EXTERN
SBMLNamespaces_t *
SBMLNamespaces_create(unsigned int level, unsigned int version)
{
  return new SBMLNamespaces(level, version);
}

LIBSBML_EXTERN
void
SBMLNamespaces_free(SBMLNamespaces_t* ns)
{
  if (ns == NULL) return;
  delete static_cast<SBMLNamespaces*>(ns);
}

LIBSBML_EXTERN
unsigned int
SBMLNamespaces_getLevel(SBMLNamespaces_t *sbmlns)
{
  return (sbmlns != NULL) ? sbmlns->getLevel() : SBML_INT_MAX;
}

LIBSBML_EXTERN
unsigned int
SBMLNamespaces_getVersion(SBMLNamespaces_t *sbmlns)
{
  return (sbmlns != NULL) ? sbmlns->getVersion() : SBML_INT_MAX;
}

LIBSBML_EXTERN
XMLNamespaces_t *
SBMLNamespaces_getNamespaces(SBMLNamespaces_t *sbmlns)
{
  return (sbmlns != NULL) ? sbmlns->getNamespaces() : NULL;
}

LIBSBML_EXTERN
char *
SBMLNamespaces_getSBMLNamespaceURI(unsigned int level, unsigned int version)
{
  return safe_strdup(SBMLNamespaces::getSBMLNamespaceURI(level, version).c_str());
}

LIBSBML_EXTERN
int
SBMLNamespaces_addNamespaces(SBMLNamespaces_t *sbmlns,
                             const XMLNamespaces_t * xmlns)
{
  if (sbmlns != NULL)
    return sbmlns->addNamespaces(xmlns);
  else
    return LIBSBML_INVALID_OBJECT;
}

LIBSBML_EXTERN
SBMLNamespaces_t **
SBMLNamespaces_getSupportedNamespaces(int *length)
{
  if (length == NULL) return NULL;
   const List* supported = SBMLNamespaces::getSupportedNamespaces();

   *length = (int) supported->getSize();
  SBMLNamespaces_t ** result = (SBMLNamespaces_t**)malloc(sizeof(SBMLNamespaces_t*)*((unsigned long)*length));
  memset(result, 0, sizeof(SBMLNamespaces_t*)*((unsigned long)*length));
  for (int i = 0; i < *length; i++)
  {
    result[i] = ((SBMLNamespaces*)supported->get((unsigned int)i))->clone();
  }
  SBMLNamespaces::freeSBMLNamespaces(const_cast<List*>(supported));
  return result;
}
/** @endcond */

LIBSBML_CPP_NAMESPACE_END

<|endoftext|>


<commit_before><commit_msg>Improve how components are displayed in node inspector<commit_after><|endoftext|>


<commit_before>/**
 * Copyright (c) 2016 zScale Technology GmbH <legal@zscale.io>
 * Authors:
 *   - Paul Asmuth <paul@zscale.io>
 *
 * This program is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License ("the license") as
 * published by the Free Software Foundation, either version 3 of the License,
 * or any later version.
 *
 * In accordance with Section 7(e) of the license, the licensing of the Program
 * under the license does not imply a trademark license. Therefore any rights,
 * title and interest in our trademarks remain entirely with us.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the license for more details.
 *
 * You can be released from the requirements of the license by purchasing a
 * commercial license. Buying such a license is mandatory as soon as you develop
 * commercial activities involving this program without disclosing the source
 * code of your own applications
 */
#include "unistd.h"
#include <eventql/util/logging.h>
#include <eventql/util/wallclock.h>
#include <eventql/util/application.h>
#include <eventql/db/ReplicationWorker.h>
#include <eventql/db/Partition.h>
#include <eventql/server/server_stats.h>

#include "eventql/eventql.h"

namespace eventql {

ReplicationInfo::ReplicationInfo() {
  reset();
}

void ReplicationInfo::reset() {
  std::unique_lock<std::mutex> lk(mutex_);
  is_idle_ = true;
  cur_partition_.clear();
  cur_target_host_.clear();
  cur_partition_since_ = UnixTime(0);
  cur_target_host_since_ = UnixTime(0);
  cur_target_host_bytes_sent_ = 0;
  cur_target_host_records_sent_ = 0;
}

void ReplicationInfo::setPartition(String name) {
  std::unique_lock<std::mutex> lk(mutex_);
  is_idle_ = false;
  cur_partition_ = name;
  cur_partition_since_ = WallClock::now();
}

void ReplicationInfo::setTargetHost(String host_name) {
  std::unique_lock<std::mutex> lk(mutex_);
  cur_target_host_ = host_name;
  cur_target_host_since_ = WallClock::now();
  cur_target_host_bytes_sent_ = 0;
  cur_target_host_records_sent_ = 0;
}

void ReplicationInfo::setTargetHostStatus(
    size_t bytes_sent,
    size_t records_sent) {
  cur_target_host_bytes_sent_ = bytes_sent;
  cur_target_host_records_sent_ = records_sent;
}

String ReplicationInfo::toString() const {
  std::unique_lock<std::mutex> lk(mutex_);
  if (is_idle_) {
    return "Idle";
  }

  if (cur_target_host_.empty()) {
    return StringUtil::format(
        "Replicating partition $0 (since $1s)",
        cur_partition_,
        (WallClock::unixMicros() - cur_partition_since_.unixMicros()) / kMicrosPerSecond);

  } else {
    auto duration = (WallClock::unixMicros() - cur_target_host_since_.unixMicros()) / kMicrosPerSecond;
    return StringUtil::format(
        "Replicating partition $0 (since $1s) to host $2 (since $3s); records_sent=$4 bytes_sent=$5MB bw=$6kb/s",
        cur_partition_,
        (WallClock::unixMicros() - cur_partition_since_.unixMicros()) / kMicrosPerSecond,
        cur_target_host_,
        duration,
        cur_target_host_records_sent_,
        cur_target_host_bytes_sent_ / double(1024 * 1024),
        (cur_target_host_bytes_sent_ / (duration < 1 ? 1 : duration)) / 1024);
  }
}

ReplicationWorker::ReplicationWorker(
    RefPtr<ReplicationScheme> repl_scheme,
    PartitionMap* pmap,
    http::HTTPConnectionPool* http) :
    repl_scheme_(repl_scheme),
    pmap_(pmap),
    http_(http),
    queue_([] (
        const Pair<uint64_t, RefPtr<Partition>>& a,
        const Pair<uint64_t, RefPtr<Partition>>& b) {
      return a.first < b.first;
    }),
    running_(false),
    num_replication_threads_(16),
    replication_infos_(num_replication_threads_) {
  pmap->subscribeToPartitionChanges([this] (
      RefPtr<eventql::PartitionChangeNotification> change) {
    enqueuePartition(change->partition);
  });

  start();
}

ReplicationWorker::~ReplicationWorker() {
  stop();
}

void ReplicationWorker::enqueuePartition(RefPtr<Partition> partition) {
  std::unique_lock<std::mutex> lk(mutex_);
  enqueuePartitionWithLock(partition, kReplicationCorkWindowMicros);
}

void ReplicationWorker::enqueuePartition(
    RefPtr<Partition> partition,
    uint64_t delay_usecs) {
  std::unique_lock<std::mutex> lk(mutex_);
  enqueuePartitionWithLock(partition, delay_usecs);
}

void ReplicationWorker::enqueuePartitionWithLock(
    RefPtr<Partition> partition,
    uint64_t delay_usecs /* = 0 */) {
  auto uuid = partition->uuid();
  if (waitset_.count(uuid) > 0) {
    return;
  }

  queue_.emplace(
      WallClock::unixMicros() + delay_usecs,
      partition);

  z1stats()->replication_queue_length.set(queue_.size());

  waitset_.emplace(uuid);
  cv_.notify_all();
}

void ReplicationWorker::start() {
  running_ = true;

  for (int i = 0; i < num_replication_threads_; ++i) {
    threads_.emplace_back(std::bind(&ReplicationWorker::work, this, i));
  }
}

void ReplicationWorker::stop() {
  if (!running_) {
    return;
  }

  running_ = false;
  cv_.notify_all();

  for (auto& t : threads_) {
    t.join();
  }
}

void ReplicationWorker::work(size_t thread_id) {
  Application::setCurrentThreadName("z1d-replication");
  auto replication_info = &replication_infos_[thread_id];

  std::unique_lock<std::mutex> lk(mutex_);

  while (running_) {
    if (queue_.size() == 0) {
      cv_.wait(lk);
    }

    if (queue_.size() == 0) {
      continue;
    }

    auto now = WallClock::unixMicros();
    if (now < queue_.begin()->first) {
      cv_.wait_for(
          lk,
          std::chrono::microseconds(queue_.begin()->first - now));

      continue;
    }

    auto partition = queue_.begin()->second;
    queue_.erase(queue_.begin());
    auto repl_scheme = repl_scheme_;

    RefPtr<PartitionReplication> repl;
    bool success = true;
    {
      lk.unlock();

      auto snap = partition->getSnapshot();
      replication_info->setPartition(StringUtil::format(
          "$0/$1/$2",
          snap->state.tsdb_namespace(),
          snap->state.table_key(),
          snap->key));

      try {
        repl = partition->getReplicationStrategy(repl_scheme, http_);
        success = repl->replicate(replication_info);
      } catch (const StandardException& e) {
        logError("evqld", e, "ReplicationWorker error");
        success = false;
      }

      replication_info->reset();

      if (!success) {
        auto snap = partition->getSnapshot();

        logError(
            "evqld",
            "Replication failed for partition $0/$1/$2",
            snap->state.tsdb_namespace(),
            snap->state.table_key(),
            snap->key.toString());
      }

      lk.lock();
    }

    if (success) {
      waitset_.erase(partition->uuid());

      repl = partition->getReplicationStrategy(repl_scheme, http_);
      if (repl->needsReplication()) {
        enqueuePartitionWithLock(partition, kReplicationCorkWindowMicros);
      } else {
        repl = partition->getReplicationStrategy(repl_scheme, http_);
        if (repl->shouldDropPartition()) {
          auto snap = partition->getSnapshot();
          auto dropped =
              pmap_->dropLocalPartition(
                  snap->state.tsdb_namespace(),
                  snap->state.table_key(),
                  snap->key);

          if (!dropped) {
            enqueuePartitionWithLock(partition, kReplicationCorkWindowMicros);
          }
        }
      }
    } else {
      auto delay = 30 * kMicrosPerSecond; // FIXPAUL increasing delay..
      queue_.emplace(now + delay, partition);
    }

    z1stats()->replication_queue_length.set(queue_.size());
  }
}

size_t ReplicationWorker::getNumThreads() const {
  return num_replication_threads_;
}

const ReplicationInfo* ReplicationWorker::getReplicationInfo(
    size_t thread_id) const {
  if (thread_id >= num_replication_threads_) {
    return nullptr;
  }

  return &replication_infos_[thread_id];
}

} // namespace eventql
<commit_msg>set # of replication threads to 8<commit_after>/**
 * Copyright (c) 2016 zScale Technology GmbH <legal@zscale.io>
 * Authors:
 *   - Paul Asmuth <paul@zscale.io>
 *
 * This program is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License ("the license") as
 * published by the Free Software Foundation, either version 3 of the License,
 * or any later version.
 *
 * In accordance with Section 7(e) of the license, the licensing of the Program
 * under the license does not imply a trademark license. Therefore any rights,
 * title and interest in our trademarks remain entirely with us.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the license for more details.
 *
 * You can be released from the requirements of the license by purchasing a
 * commercial license. Buying such a license is mandatory as soon as you develop
 * commercial activities involving this program without disclosing the source
 * code of your own applications
 */
#include "unistd.h"
#include <eventql/util/logging.h>
#include <eventql/util/wallclock.h>
#include <eventql/util/application.h>
#include <eventql/db/ReplicationWorker.h>
#include <eventql/db/Partition.h>
#include <eventql/server/server_stats.h>

#include "eventql/eventql.h"

namespace eventql {

ReplicationInfo::ReplicationInfo() {
  reset();
}

void ReplicationInfo::reset() {
  std::unique_lock<std::mutex> lk(mutex_);
  is_idle_ = true;
  cur_partition_.clear();
  cur_target_host_.clear();
  cur_partition_since_ = UnixTime(0);
  cur_target_host_since_ = UnixTime(0);
  cur_target_host_bytes_sent_ = 0;
  cur_target_host_records_sent_ = 0;
}

void ReplicationInfo::setPartition(String name) {
  std::unique_lock<std::mutex> lk(mutex_);
  is_idle_ = false;
  cur_partition_ = name;
  cur_partition_since_ = WallClock::now();
}

void ReplicationInfo::setTargetHost(String host_name) {
  std::unique_lock<std::mutex> lk(mutex_);
  cur_target_host_ = host_name;
  cur_target_host_since_ = WallClock::now();
  cur_target_host_bytes_sent_ = 0;
  cur_target_host_records_sent_ = 0;
}

void ReplicationInfo::setTargetHostStatus(
    size_t bytes_sent,
    size_t records_sent) {
  cur_target_host_bytes_sent_ = bytes_sent;
  cur_target_host_records_sent_ = records_sent;
}

String ReplicationInfo::toString() const {
  std::unique_lock<std::mutex> lk(mutex_);
  if (is_idle_) {
    return "Idle";
  }

  if (cur_target_host_.empty()) {
    return StringUtil::format(
        "Replicating partition $0 (since $1s)",
        cur_partition_,
        (WallClock::unixMicros() - cur_partition_since_.unixMicros()) / kMicrosPerSecond);

  } else {
    auto duration = (WallClock::unixMicros() - cur_target_host_since_.unixMicros()) / kMicrosPerSecond;
    return StringUtil::format(
        "Replicating partition $0 (since $1s) to host $2 (since $3s); records_sent=$4 bytes_sent=$5MB bw=$6kb/s",
        cur_partition_,
        (WallClock::unixMicros() - cur_partition_since_.unixMicros()) / kMicrosPerSecond,
        cur_target_host_,
        duration,
        cur_target_host_records_sent_,
        cur_target_host_bytes_sent_ / double(1024 * 1024),
        (cur_target_host_bytes_sent_ / (duration < 1 ? 1 : duration)) / 1024);
  }
}

ReplicationWorker::ReplicationWorker(
    RefPtr<ReplicationScheme> repl_scheme,
    PartitionMap* pmap,
    http::HTTPConnectionPool* http) :
    repl_scheme_(repl_scheme),
    pmap_(pmap),
    http_(http),
    queue_([] (
        const Pair<uint64_t, RefPtr<Partition>>& a,
        const Pair<uint64_t, RefPtr<Partition>>& b) {
      return a.first < b.first;
    }),
    running_(false),
    num_replication_threads_(8),
    replication_infos_(num_replication_threads_) {
  pmap->subscribeToPartitionChanges([this] (
      RefPtr<eventql::PartitionChangeNotification> change) {
    enqueuePartition(change->partition);
  });

  start();
}

ReplicationWorker::~ReplicationWorker() {
  stop();
}

void ReplicationWorker::enqueuePartition(RefPtr<Partition> partition) {
  std::unique_lock<std::mutex> lk(mutex_);
  enqueuePartitionWithLock(partition, kReplicationCorkWindowMicros);
}

void ReplicationWorker::enqueuePartition(
    RefPtr<Partition> partition,
    uint64_t delay_usecs) {
  std::unique_lock<std::mutex> lk(mutex_);
  enqueuePartitionWithLock(partition, delay_usecs);
}

void ReplicationWorker::enqueuePartitionWithLock(
    RefPtr<Partition> partition,
    uint64_t delay_usecs /* = 0 */) {
  auto uuid = partition->uuid();
  if (waitset_.count(uuid) > 0) {
    return;
  }

  queue_.emplace(
      WallClock::unixMicros() + delay_usecs,
      partition);

  z1stats()->replication_queue_length.set(queue_.size());

  waitset_.emplace(uuid);
  cv_.notify_all();
}

void ReplicationWorker::start() {
  running_ = true;

  for (int i = 0; i < num_replication_threads_; ++i) {
    threads_.emplace_back(std::bind(&ReplicationWorker::work, this, i));
  }
}

void ReplicationWorker::stop() {
  if (!running_) {
    return;
  }

  running_ = false;
  cv_.notify_all();

  for (auto& t : threads_) {
    t.join();
  }
}

void ReplicationWorker::work(size_t thread_id) {
  Application::setCurrentThreadName("z1d-replication");
  auto replication_info = &replication_infos_[thread_id];

  std::unique_lock<std::mutex> lk(mutex_);

  while (running_) {
    if (queue_.size() == 0) {
      cv_.wait(lk);
    }

    if (queue_.size() == 0) {
      continue;
    }

    auto now = WallClock::unixMicros();
    if (now < queue_.begin()->first) {
      cv_.wait_for(
          lk,
          std::chrono::microseconds(queue_.begin()->first - now));

      continue;
    }

    auto partition = queue_.begin()->second;
    queue_.erase(queue_.begin());
    auto repl_scheme = repl_scheme_;

    RefPtr<PartitionReplication> repl;
    bool success = true;
    {
      lk.unlock();

      auto snap = partition->getSnapshot();
      replication_info->setPartition(StringUtil::format(
          "$0/$1/$2",
          snap->state.tsdb_namespace(),
          snap->state.table_key(),
          snap->key));

      try {
        repl = partition->getReplicationStrategy(repl_scheme, http_);
        success = repl->replicate(replication_info);
      } catch (const StandardException& e) {
        logError("evqld", e, "ReplicationWorker error");
        success = false;
      }

      replication_info->reset();

      if (!success) {
        auto snap = partition->getSnapshot();

        logError(
            "evqld",
            "Replication failed for partition $0/$1/$2",
            snap->state.tsdb_namespace(),
            snap->state.table_key(),
            snap->key.toString());
      }

      lk.lock();
    }

    if (success) {
      waitset_.erase(partition->uuid());

      repl = partition->getReplicationStrategy(repl_scheme, http_);
      if (repl->needsReplication()) {
        enqueuePartitionWithLock(partition, kReplicationCorkWindowMicros);
      } else {
        repl = partition->getReplicationStrategy(repl_scheme, http_);
        if (repl->shouldDropPartition()) {
          auto snap = partition->getSnapshot();
          auto dropped =
              pmap_->dropLocalPartition(
                  snap->state.tsdb_namespace(),
                  snap->state.table_key(),
                  snap->key);

          if (!dropped) {
            enqueuePartitionWithLock(partition, kReplicationCorkWindowMicros);
          }
        }
      }
    } else {
      auto delay = 30 * kMicrosPerSecond; // FIXPAUL increasing delay..
      queue_.emplace(now + delay, partition);
    }

    z1stats()->replication_queue_length.set(queue_.size());
  }
}

size_t ReplicationWorker::getNumThreads() const {
  return num_replication_threads_;
}

const ReplicationInfo* ReplicationWorker::getReplicationInfo(
    size_t thread_id) const {
  if (thread_id >= num_replication_threads_) {
    return nullptr;
  }

  return &replication_infos_[thread_id];
}

} // namespace eventql
<|endoftext|>


<commit_before>#include <QApplication>
#include <QDir>
#include <QFileInfo>
#include <QDateTime>
#include <QTimer>
#include <QThreadPool>

#include "seafile-applet.h"
#include "ui/tray-icon.h"
#include "configurator.h"
#include "account-mgr.h"
#include "rpc/rpc-client.h"
#include "rpc/local-repo.h"
#include "utils/file-utils.h"
#include "utils/utils.h"
#include "utils/uninstall-helpers.h"
#include "data-mgr.h"
#include "tasks.h"
#include "transfer-mgr.h"

#include "auto-update-mgr.h"

namespace {

const char *kFileCacheTopDirName = "file-cache";
const char *kFileCacheTempTopDirName = "file-cache-tmp";

inline bool addPath(QFileSystemWatcher *watcher, const QString &file) {
  if (watcher->files().contains(file))
      return true;
  bool ret = watcher->addPath(file);
  if (!ret) {
      qWarning("[AutoUpdateManager] failed to watch cache file %s", file.toUtf8().data());
  }
  return ret;
}

inline bool removePath(QFileSystemWatcher *watcher, const QString &file) {
  if (!watcher->files().contains(file))
      return true;
  bool ret = watcher->removePath(file);
  if (!ret) {
      qWarning("[AutoUpdateManager] failed to remove watch on cache file %s", file.toUtf8().data());
  }
  return ret;
}
} // anonymous namespace

SINGLETON_IMPL(AutoUpdateManager)

AutoUpdateManager::AutoUpdateManager()
{
    system_shut_down_ = false;
    connect(&watcher_, SIGNAL(fileChanged(const QString&)),
            this, SLOT(onFileChanged(const QString&)));
    connect(qApp, SIGNAL(aboutToQuit()),
            this, SLOT(systemShutDown()));
}

void AutoUpdateManager::systemShutDown()
{
    system_shut_down_ = true;
}

void AutoUpdateManager::start()
{
    cleanCachedFile();
}

void AutoUpdateManager::watchCachedFile(const Account& account,
                                        const QString& repo_id,
                                        const QString& path)
{
    QString local_path = DataManager::getLocalCacheFilePath(repo_id, path);
    qDebug("[AutoUpdateManager] watch cache file %s", local_path.toUtf8().data());
    if (!QFileInfo(local_path).exists()) {
        qWarning("[AutoUpdateManager] unable to watch non-existent cache file %s", local_path.toUtf8().data());
        return;
    }

    // do we have it in deferred list ?
    // skip if yes
    Q_FOREACH(const WatchedFileInfo& info, deleted_files_infos_)
    {
        if (repo_id == info.repo_id && path == info.path_in_repo)
            return;
    }

    addPath(&watcher_, local_path);

    QFileInfo finfo(local_path);
    watch_infos_[local_path] =
        WatchedFileInfo(account,
                        repo_id,
                        path,
                        finfo.lastModified().toMSecsSinceEpoch(),
                        finfo.size());
}

void AutoUpdateManager::cleanCachedFile()
{
    qDebug("[AutoUpdateManager] cancel all download tasks");
    TransferManager::instance()->cancelAllDownloadTasks();

    const Account cur_account = seafApplet->accountManager()->currentAccount();
    foreach(const QString& key, watch_infos_.keys()) {
        if (watch_infos_[key].account == cur_account)
            watch_infos_.remove(key);
    }

    FileCache::instance()->cleanCurrentAccountCache();

    CachedFilesCleaner *cleaner = new CachedFilesCleaner();
    QThreadPool::globalInstance()->start(cleaner);
}

void AutoUpdateManager::uploadFile(const QString& local_path)
{
    WatchedFileInfo &info = watch_infos_[local_path];

    FileNetworkTask *task = seafApplet->dataManager()->createUploadTask(
        info.repo_id, ::getParentPath(info.path_in_repo),
        local_path, ::getBaseName(local_path), true);

    connect(task, SIGNAL(finished(bool)),
            this, SLOT(onUpdateTaskFinished(bool)));

    qDebug("[AutoUpdateManager] start uploading new version of file %s", local_path.toUtf8().data());

    info.uploading = true;

    task->start();
}

void AutoUpdateManager::onFileChanged(const QString& local_path)
{
    qDebug("[AutoUpdateManager] detected cache file %s changed", local_path.toUtf8().data());
    if (!watch_infos_.contains(local_path)) {
        // filter unwanted events
        return;
    }

    WatchedFileInfo &info = watch_infos_[local_path];
    QFileInfo finfo(local_path);

#ifdef Q_OS_MAC
    if (MacImageFilesWorkAround::instance()->isRecentOpenedImage(local_path)) {
        qDebug("[AutoUpdateManager] skip the image file updates on mac for %s", toCStr(local_path));
        return;
    }
#endif
    removePath(&watcher_, local_path);
    QString repo_id, path_in_repo;

    if (!finfo.exists()) {
        qDebug("[AutoUpdateManager] detected cache file %s renamed or removed", local_path.toUtf8().data());
        WatchedFileInfo deferred_info = info;
        removeWatch(local_path);
        // Some application would deleted and recreate the file when saving.
        // We work around that by double checking whether the file gets
        // recreated after a short period
        QTimer::singleShot(5000, this, SLOT(checkFileRecreated()));
        deleted_files_infos_.enqueue(deferred_info);
        return;
    }

    uploadFile(local_path);
}

void AutoUpdateManager::onUpdateTaskFinished(bool success)
{
    if (system_shut_down_) {
        return;
    }

    FileUploadTask *task = qobject_cast<FileUploadTask *>(sender());
    if (task == NULL)
        return;
    const QString local_path = task->localFilePath();
    const QFileInfo finfo = QFileInfo(local_path);
    if (!finfo.exists()) {
        //TODO: What if the delete&recreate happens just before this function is called?
        qWarning("[AutoUpdateManager] file %s not exists anymore", toCStr(local_path));
        return;
    }

    if (!watch_infos_.contains(local_path)) {
        qWarning("[AutoUpdateManager] no watch info for file %s", toCStr(local_path));
        return;
    }
    WatchedFileInfo& info = watch_infos_[local_path];
    info.uploading = false;

    if (success) {
        qDebug("[AutoUpdateManager] uploaded new version of file %s", local_path.toUtf8().data());
        seafApplet->trayIcon()->showMessage(tr("Upload Success"),
                                            tr("File \"%1\"\nuploaded successfully.").arg(finfo.fileName()),
                                            task->repoId());

        // This would also set the "uploading" and "num_upload_errors" column to 0.
        FileCache::instance()->saveCachedFileId(task->repoId(),
                                                info.path_in_repo,
                                                task->account().getSignature(),
                                                task->oid(),
                                                task->localFilePath());
        emit fileUpdated(task->repoId(), task->path());
    } else {
        qWarning("[AutoUpdateManager] failed to upload new version of file %s: %s",
                 toCStr(local_path),
                 toCStr(task->errorString()));
        QString error_msg;
        if (task->httpErrorCode() == 403) {
            error_msg = tr("Permission Error!");
        } else if (task->httpErrorCode() == 401) {
            error_msg = tr("Authorization expired");
        } else if (task->httpErrorCode() == 441) {
            error_msg = tr("File does not exist");
        } else {
            error_msg = task->errorString();
        }

        QString name = ::getBaseName(local_path);
        DirentsCache::ReturnEntry retval = DirentsCache::instance()->getCachedDirents(info.repo_id, task->path());
        QList<SeafDirent> *l = retval.second;
        QString msg = tr("File \"%1\"\nfailed to upload.").arg(QFileInfo(local_path).fileName());
        if (l != NULL) {
            foreach (const SeafDirent dirent, *l) {
                if (dirent.name == name) {
                    if (dirent.is_locked) {
                        msg = tr("The file is locked by %1, "
                                 "please try again later").arg(dirent.getLockOwnerDisplayString());
                    }
                }
            }
        }
        seafApplet->trayIcon()->showMessage(tr("Upload Failure: %1").arg(error_msg),
                                            msg,
                                            task->repoId());
    }

    addPath(&watcher_, local_path);
}

void AutoUpdateManager::removeWatch(const QString& local_path)
{
    watch_infos_.remove(local_path);
    removePath(&watcher_, local_path);
}

void AutoUpdateManager::checkFileRecreated()
{
    if (deleted_files_infos_.isEmpty()) {
        // impossible
        return;
    }

    const WatchedFileInfo info = deleted_files_infos_.dequeue();
    const QString path = DataManager::getLocalCacheFilePath(info.repo_id, info.path_in_repo);
    if (QFileInfo(path).exists()) {
        qDebug("[AutoUpdateManager] detected recreated file %s", path.toUtf8().data());
        addPath(&watcher_, path);
        watch_infos_[path] = info;
        // Some applications like MSOffice would remove the original file and
        // recreate it when the user modifies the file.
        onFileChanged(path);
    }
}

QHash<QString, AutoUpdateManager::FileStatus>
AutoUpdateManager::getFileStatusForDirectory(const QString &account_sig,
                                             const QString &repo_id,
                                             const QString &parent_dir,
                                             const QList<SeafDirent>& dirents)
{
    QHash<QString, SeafDirent> dirents_map;
    foreach(const SeafDirent& d, dirents) {
        if (d.isFile()) {
            dirents_map[d.name] = d;
        }
    }

    QHash<QString, FileStatus> ret;
    QList<FileCache::CacheEntry> caches =
        FileCache::instance()->getCachedFilesForDirectory(account_sig, repo_id, parent_dir);
    if (caches.empty()) {
        // qDebug("no cached files for dir %s\n", toCStr(parent_dir));
    }
    foreach(const FileCache::CacheEntry& entry, caches) {
        // qDebug("found cache entry: %s\n", entry.path.toUtf8().data());

        QString local_file_path = DataManager::getLocalCacheFilePath(entry.repo_id, entry.path);
        const QString& file = ::getBaseName(entry.path);
        bool is_uploading = watch_infos_.contains(local_file_path) && watch_infos_[local_file_path].uploading;

        if (!dirents_map.contains(file)) {
            // qDebug("cached files no longer exists: %s\n", entry.path.toUtf8().data());
            continue;
        }

        const SeafDirent& d = dirents_map[file];
        if (d.id != entry.file_id) {
            // qDebug("cached file is a stale version: %s\n", entry.path.toUtf8().data());
            ret[file] = is_uploading ? UPLOADING : NOT_SYNCED;
            continue;
        }

        QFileInfo finfo(local_file_path);

        qint64 mtime = finfo.lastModified().toMSecsSinceEpoch();
        bool consistent = mtime == entry.seafile_mtime && finfo.size() == entry.seafile_size;
        if (consistent) {
            ret[file] = SYNCED;
        } else {
            ret[file] = is_uploading ? UPLOADING : NOT_SYNCED;
        }
    }
    return ret;
}

#ifdef Q_OS_MAC
SINGLETON_IMPL(MacImageFilesWorkAround)

MacImageFilesWorkAround::MacImageFilesWorkAround()
{
}

void MacImageFilesWorkAround::fileOpened(const QString& path)
{
    QString mimetype = ::mimeTypeFromFileName(path);
    if (mimetype.startsWith("image") || mimetype == "application/pdf") {
        images_[path] = QDateTime::currentMSecsSinceEpoch();
    }
}

bool MacImageFilesWorkAround::isRecentOpenedImage(const QString& path)
{
    qint64 ts = images_.value(path, 0);
    if (QDateTime::currentMSecsSinceEpoch() < ts + 1000 * 10) {
        return true;
    } else {
        return false;
    }
}
#endif

CachedFilesCleaner::CachedFilesCleaner()
{
}

void CachedFilesCleaner::run()
{
    QString file_cache_dir = pathJoin(seafApplet->configurator()->seafileDir(),
                               kFileCacheTopDirName);
    QString file_cache_tmp_dir = pathJoin(seafApplet->configurator()->seafileDir(),
                                   kFileCacheTempTopDirName);

    qDebug("[AutoUpdateManager] removing cached files");
    if (QDir(file_cache_tmp_dir).exists()) {
        delete_dir_recursively(file_cache_tmp_dir);
    }
    if (QDir(file_cache_dir).exists()) {
        QDir().rename(file_cache_dir, file_cache_tmp_dir);
        delete_dir_recursively(file_cache_tmp_dir);
    }
}
<commit_msg>Repair file auto upload after download doc file (#1041)<commit_after>#include <QApplication>
#include <QDir>
#include <QFileInfo>
#include <QDateTime>
#include <QTimer>
#include <QThreadPool>

#include "seafile-applet.h"
#include "ui/tray-icon.h"
#include "configurator.h"
#include "account-mgr.h"
#include "rpc/rpc-client.h"
#include "rpc/local-repo.h"
#include "utils/file-utils.h"
#include "utils/utils.h"
#include "utils/uninstall-helpers.h"
#include "data-mgr.h"
#include "tasks.h"
#include "transfer-mgr.h"

#include "auto-update-mgr.h"

namespace {

const char *kFileCacheTopDirName = "file-cache";
const char *kFileCacheTempTopDirName = "file-cache-tmp";

inline bool addPath(QFileSystemWatcher *watcher, const QString &file) {
  if (watcher->files().contains(file))
      return true;
  bool ret = watcher->addPath(file);
  if (!ret) {
      qWarning("[AutoUpdateManager] failed to watch cache file %s", file.toUtf8().data());
  }
  return ret;
}

inline bool removePath(QFileSystemWatcher *watcher, const QString &file) {
  if (!watcher->files().contains(file))
      return true;
  bool ret = watcher->removePath(file);
  if (!ret) {
      qWarning("[AutoUpdateManager] failed to remove watch on cache file %s", file.toUtf8().data());
  }
  return ret;
}
} // anonymous namespace

SINGLETON_IMPL(AutoUpdateManager)

AutoUpdateManager::AutoUpdateManager()
{
    system_shut_down_ = false;
    connect(&watcher_, SIGNAL(fileChanged(const QString&)),
            this, SLOT(onFileChanged(const QString&)));
    connect(qApp, SIGNAL(aboutToQuit()),
            this, SLOT(systemShutDown()));
}

void AutoUpdateManager::systemShutDown()
{
    system_shut_down_ = true;
}

void AutoUpdateManager::start()
{
    cleanCachedFile();
}

void AutoUpdateManager::watchCachedFile(const Account& account,
                                        const QString& repo_id,
                                        const QString& path)
{
    QString local_path = DataManager::getLocalCacheFilePath(repo_id, path);
    qDebug("[AutoUpdateManager] watch cache file %s", local_path.toUtf8().data());
    if (!QFileInfo(local_path).exists()) {
        qWarning("[AutoUpdateManager] unable to watch non-existent cache file %s", local_path.toUtf8().data());
        return;
    }

    // do we have it in deferred list ?
    // skip if yes
    Q_FOREACH(const WatchedFileInfo& info, deleted_files_infos_)
    {
        if (repo_id == info.repo_id && path == info.path_in_repo)
            return;
    }

    addPath(&watcher_, local_path);

    QFileInfo finfo(local_path);
    watch_infos_[local_path] =
        WatchedFileInfo(account,
                        repo_id,
                        path,
                        finfo.lastModified().toMSecsSinceEpoch(),
                        finfo.size());
}

void AutoUpdateManager::cleanCachedFile()
{
    qDebug("[AutoUpdateManager] cancel all download tasks");
    TransferManager::instance()->cancelAllDownloadTasks();

    const Account cur_account = seafApplet->accountManager()->currentAccount();
    foreach(const QString& key, watch_infos_.keys()) {
        if (watch_infos_[key].account == cur_account)
            watch_infos_.remove(key);
    }

    FileCache::instance()->cleanCurrentAccountCache();

    CachedFilesCleaner *cleaner = new CachedFilesCleaner();
    QThreadPool::globalInstance()->start(cleaner);
}

void AutoUpdateManager::uploadFile(const QString& local_path)
{
    WatchedFileInfo &info = watch_infos_[local_path];

    FileNetworkTask *task = seafApplet->dataManager()->createUploadTask(
        info.repo_id, ::getParentPath(info.path_in_repo),
        local_path, ::getBaseName(local_path), true);

    connect(task, SIGNAL(finished(bool)),
            this, SLOT(onUpdateTaskFinished(bool)));

    qDebug("[AutoUpdateManager] start uploading new version of file %s", local_path.toUtf8().data());

    info.uploading = true;

    task->start();
}

void AutoUpdateManager::onFileChanged(const QString& local_path)
{
    qDebug("[AutoUpdateManager] detected cache file %s changed", local_path.toUtf8().data());
    if (!watch_infos_.contains(local_path)) {
        // filter unwanted events
        return;
    }

    WatchedFileInfo &info = watch_infos_[local_path];
    QFileInfo finfo(local_path);

    // Download the doc file in the mac will automatically upload
    // If the timestamp has not changed, it will not be uploaded
    qint64 mtime = finfo.lastModified().toMSecsSinceEpoch();
    if (mtime == info.mtime) {
        qDebug("[AutoUpdateManager] Received a file %s upload notification, but the timestamp has not changed, "
               "it will not upload", local_path.toUtf8().data());
        return;
    }

#ifdef Q_OS_MAC
    if (MacImageFilesWorkAround::instance()->isRecentOpenedImage(local_path)) {
        qDebug("[AutoUpdateManager] skip the image file updates on mac for %s", toCStr(local_path));
        return;
    }
#endif
    removePath(&watcher_, local_path);
    QString repo_id, path_in_repo;

    if (!finfo.exists()) {
        qDebug("[AutoUpdateManager] detected cache file %s renamed or removed", local_path.toUtf8().data());
        WatchedFileInfo deferred_info = info;
        removeWatch(local_path);
        // Some application would deleted and recreate the file when saving.
        // We work around that by double checking whether the file gets
        // recreated after a short period
        QTimer::singleShot(5000, this, SLOT(checkFileRecreated()));
        deleted_files_infos_.enqueue(deferred_info);
        return;
    }

    uploadFile(local_path);
}

void AutoUpdateManager::onUpdateTaskFinished(bool success)
{
    if (system_shut_down_) {
        return;
    }

    FileUploadTask *task = qobject_cast<FileUploadTask *>(sender());
    if (task == NULL)
        return;
    const QString local_path = task->localFilePath();
    const QFileInfo finfo = QFileInfo(local_path);
    if (!finfo.exists()) {
        //TODO: What if the delete&recreate happens just before this function is called?
        qWarning("[AutoUpdateManager] file %s not exists anymore", toCStr(local_path));
        return;
    }

    if (!watch_infos_.contains(local_path)) {
        qWarning("[AutoUpdateManager] no watch info for file %s", toCStr(local_path));
        return;
    }
    WatchedFileInfo& info = watch_infos_[local_path];
    info.uploading = false;

    if (success) {
        qDebug("[AutoUpdateManager] uploaded new version of file %s", local_path.toUtf8().data());
        seafApplet->trayIcon()->showMessage(tr("Upload Success"),
                                            tr("File \"%1\"\nuploaded successfully.").arg(finfo.fileName()),
                                            task->repoId());

        // This would also set the "uploading" and "num_upload_errors" column to 0.
        FileCache::instance()->saveCachedFileId(task->repoId(),
                                                info.path_in_repo,
                                                task->account().getSignature(),
                                                task->oid(),
                                                task->localFilePath());
        emit fileUpdated(task->repoId(), task->path());
    } else {
        qWarning("[AutoUpdateManager] failed to upload new version of file %s: %s",
                 toCStr(local_path),
                 toCStr(task->errorString()));
        QString error_msg;
        if (task->httpErrorCode() == 403) {
            error_msg = tr("Permission Error!");
        } else if (task->httpErrorCode() == 401) {
            error_msg = tr("Authorization expired");
        } else if (task->httpErrorCode() == 441) {
            error_msg = tr("File does not exist");
        } else {
            error_msg = task->errorString();
        }

        QString name = ::getBaseName(local_path);
        DirentsCache::ReturnEntry retval = DirentsCache::instance()->getCachedDirents(info.repo_id, task->path());
        QList<SeafDirent> *l = retval.second;
        QString msg = tr("File \"%1\"\nfailed to upload.").arg(QFileInfo(local_path).fileName());
        if (l != NULL) {
            foreach (const SeafDirent dirent, *l) {
                if (dirent.name == name) {
                    if (dirent.is_locked) {
                        msg = tr("The file is locked by %1, "
                                 "please try again later").arg(dirent.getLockOwnerDisplayString());
                    }
                }
            }
        }
        seafApplet->trayIcon()->showMessage(tr("Upload Failure: %1").arg(error_msg),
                                            msg,
                                            task->repoId());
    }

    addPath(&watcher_, local_path);
}

void AutoUpdateManager::removeWatch(const QString& local_path)
{
    watch_infos_.remove(local_path);
    removePath(&watcher_, local_path);
}

void AutoUpdateManager::checkFileRecreated()
{
    if (deleted_files_infos_.isEmpty()) {
        // impossible
        return;
    }

    const WatchedFileInfo info = deleted_files_infos_.dequeue();
    const QString path = DataManager::getLocalCacheFilePath(info.repo_id, info.path_in_repo);
    if (QFileInfo(path).exists()) {
        qDebug("[AutoUpdateManager] detected recreated file %s", path.toUtf8().data());
        addPath(&watcher_, path);
        watch_infos_[path] = info;
        // Some applications like MSOffice would remove the original file and
        // recreate it when the user modifies the file.
        onFileChanged(path);
    }
}

QHash<QString, AutoUpdateManager::FileStatus>
AutoUpdateManager::getFileStatusForDirectory(const QString &account_sig,
                                             const QString &repo_id,
                                             const QString &parent_dir,
                                             const QList<SeafDirent>& dirents)
{
    QHash<QString, SeafDirent> dirents_map;
    foreach(const SeafDirent& d, dirents) {
        if (d.isFile()) {
            dirents_map[d.name] = d;
        }
    }

    QHash<QString, FileStatus> ret;
    QList<FileCache::CacheEntry> caches =
        FileCache::instance()->getCachedFilesForDirectory(account_sig, repo_id, parent_dir);
    if (caches.empty()) {
        // qDebug("no cached files for dir %s\n", toCStr(parent_dir));
    }
    foreach(const FileCache::CacheEntry& entry, caches) {
        // qDebug("found cache entry: %s\n", entry.path.toUtf8().data());

        QString local_file_path = DataManager::getLocalCacheFilePath(entry.repo_id, entry.path);
        const QString& file = ::getBaseName(entry.path);
        bool is_uploading = watch_infos_.contains(local_file_path) && watch_infos_[local_file_path].uploading;

        if (!dirents_map.contains(file)) {
            // qDebug("cached files no longer exists: %s\n", entry.path.toUtf8().data());
            continue;
        }

        const SeafDirent& d = dirents_map[file];
        if (d.id != entry.file_id) {
            // qDebug("cached file is a stale version: %s\n", entry.path.toUtf8().data());
            ret[file] = is_uploading ? UPLOADING : NOT_SYNCED;
            continue;
        }

        QFileInfo finfo(local_file_path);

        qint64 mtime = finfo.lastModified().toMSecsSinceEpoch();
        bool consistent = mtime == entry.seafile_mtime && finfo.size() == entry.seafile_size;
        if (consistent) {
            ret[file] = SYNCED;
        } else {
            ret[file] = is_uploading ? UPLOADING : NOT_SYNCED;
        }
    }
    return ret;
}

#ifdef Q_OS_MAC
SINGLETON_IMPL(MacImageFilesWorkAround)

MacImageFilesWorkAround::MacImageFilesWorkAround()
{
}

void MacImageFilesWorkAround::fileOpened(const QString& path)
{
    QString mimetype = ::mimeTypeFromFileName(path);
    if (mimetype.startsWith("image") || mimetype == "application/pdf") {
        images_[path] = QDateTime::currentMSecsSinceEpoch();
    }
}

bool MacImageFilesWorkAround::isRecentOpenedImage(const QString& path)
{
    qint64 ts = images_.value(path, 0);
    if (QDateTime::currentMSecsSinceEpoch() < ts + 1000 * 10) {
        return true;
    } else {
        return false;
    }
}
#endif

CachedFilesCleaner::CachedFilesCleaner()
{
}

void CachedFilesCleaner::run()
{
    QString file_cache_dir = pathJoin(seafApplet->configurator()->seafileDir(),
                               kFileCacheTopDirName);
    QString file_cache_tmp_dir = pathJoin(seafApplet->configurator()->seafileDir(),
                                   kFileCacheTempTopDirName);

    qDebug("[AutoUpdateManager] removing cached files");
    if (QDir(file_cache_tmp_dir).exists()) {
        delete_dir_recursively(file_cache_tmp_dir);
    }
    if (QDir(file_cache_dir).exists()) {
        QDir().rename(file_cache_dir, file_cache_tmp_dir);
        delete_dir_recursively(file_cache_tmp_dir);
    }
}
<|endoftext|>


<commit_before>/**
 * Appcelerator Titanium - licensed under the Apache Public License 2
 * see LICENSE in the root folder for details on the license.
 * Copyright (c) 2009 Appcelerator, Inc. All Rights Reserved.
 */
#include <vector>
#include "process.h"
#include "pipe.h"
#include <Poco/Path.h>
#include <Poco/RunnableAdapter.h>
#include <Poco/ScopedLock.h>

using Poco::RunnableAdapter;

namespace ti
{
	Process::Process(ProcessBinding* parent, std::string& cmd, std::vector<std::string>& args) :
		running(false),
		complete(false),
		pid(-1),
		exitCode(-1),
		errp(0),
		outp(0),
		inp(0),
		logger(Logger::Get("Process.Process"))
	{
		this->parent = parent;
		this->errp = new Poco::Pipe();
		this->outp = new Poco::Pipe();
		this->inp = new Poco::Pipe();

		logger->Debug("Running process %s", cmd.c_str());
		try
		{
			this->arguments = args;
			this->command = cmd;
		}
		catch (std::exception &e)
		{
			throw ValueException::FromString(e.what());
		}

		/**
		 * @tiapi(property=True,type=string,name=Process.Process.command,version=0.2)
		 * @tiapi The command used for the Process object
		 */
		this->SetString("command", cmd);

		/**
		 * @tiapi(property=True,type=integer,name=Process.Process.pid,version=0.2)
		 * @tiapi The process id of the Process object
		 */
		this->SetNull("pid");

		/**
		 * @tiapi(property=True,type=boolean,name=Process.Process.running,version=0.2)
		 * @tiapi The running status of the Process object
		 */
		this->SetBool("running", false);

		/**
		 * @tiapi(property=True,type=object,name=Process.Process.err,version=0.2)
		 * @tiapi The Pipe object of the error stream
		 */
		this->err = new Pipe(new Poco::PipeInputStream(*errp));
		this->shared_error = new SharedKObject(this->err);
		this->SetObject("err", *shared_error);

		/**
		 * @tiapi(property=True,type=object,name=Process.Process.out,version=0.2)
		 * @tiapi The Pipe object of the output stream
		 */
		this->out = new Pipe(new Poco::PipeInputStream(*outp));
		this->shared_output = new SharedKObject(this->out);
		this->SetObject("out", *shared_output);

		/**
		 * @tiapi(property=True,type=object,name=Process.Process.in,version=0.2)
		 * @tiapi The Pipe object of the input stream
		 */
		this->in = new Pipe(new Poco::PipeOutputStream(*inp));
		this->shared_input = new SharedKObject(this->in);
		this->SetObject("in", *shared_input);

		/**
		 * @tiapi(method=True,name=Process.Process.terminate,version=0.2)
		 * @tiapi Terminates a running process
		 */
		this->SetMethod("terminate", &Process::Terminate);

		/**
		 * @tiapi(property=True,type=integer,name=Process.Process.exitCode,version=0.4)
		 * @tiapi The exit code or null if not yet exited
		 */
		this->SetNull("exitCode");

		/**
		 * @tiapi(property=True,type=method,name=Process.Process.onread,since=0.4)
		 * @tiapi The function handler to call when sys out is read
		 */
		this->SetNull("onread");

		/**
		 * @tiapi(property=True,type=method,name=Process.Process.onexit,since=0.4)
		 * @tiapi The function handler to call when the process exits
		 */
		this->SetNull("onexit");

		// setup threads which can read output and also monitor the exit
		this->monitorAdapter = new RunnableAdapter<Process>(*this, &Process::Monitor);
		this->exitMonitorThread.start(*monitorAdapter);
	}

	Process::~Process()
	{
		Terminate();

		if (this->exitMonitorThread.isRunning())
		{
			try
			{
				this->exitMonitorThread.join();
			}
			catch (Poco::Exception& e)
			{
				logger->Error(
					"Exception while try to join with exit monitor thread: %s",
					e.displayText().c_str());
			}
		}

		if (this->stdOutThread.isRunning())
		{
			try
			{
				this->stdOutThread.join();
			}
			catch (Poco::Exception& e)
			{
				logger->Error(
					"Exception while try to join with stdout thread: %s",
					e.displayText().c_str());
			}
		}

		if (this->stdErrorThread.isRunning())
		{
			try
			{
				this->stdErrorThread.join();
			}
			catch (Poco::Exception& e)
			{
				logger->Error(
					"Exception while try to join with stderr thread: %s",
					e.displayText().c_str());
			}
		}

		delete monitorAdapter;
		delete stdOutAdapter;
		delete stdErrorAdapter;
		delete shared_output;
		delete shared_input;
		delete shared_error;
	}

	void Process::StartReadThreads()
	{
		this->logger->Debug("Starting output handler threads...");
		if (!stdOutThread.isRunning())
		{
			this->stdOutAdapter = new RunnableAdapter<Process>(*this, &Process::ReadStdOut);
			stdOutThread.start(*stdOutAdapter);
		}

		if (!stdErrorThread.isRunning())
		{
			RunnableAdapter<Process> adapter(*this, &Process::ReadStdError);
			this->stdErrorAdapter = new RunnableAdapter<Process>(*this, &Process::ReadStdError);
			stdErrorThread.start(*stdErrorAdapter);
		}
	}

	void Process::Monitor()
	{
		this->Set("running", Value::NewBool(true));
		this->running = true;

		try
		{
			Poco::ProcessHandle ph = Poco::Process::launch(
				this->command, this->arguments,
				this->inp, this->outp, this->errp);
			this->StartReadThreads();
			this->pid = (int) ph.id();
			this->Set("pid", Value::NewInt(this->pid));

			this->exitCode = ph.wait();
			this->Set("exitCode", Value::NewInt(this->exitCode));
			logger->Debug("%s exited with return code %i",
				this->command.c_str(), this->exitCode);
		}
		catch (Poco::SystemException &se)
		{
			logger->Error("System Exception starting: %s, message: %s",
				this->command.c_str(), se.what());
		}
		catch (std::exception &e)
		{
			logger->Error("Exception starting: %s, message: %s",
				this->command.c_str(), e.what());
		}

		this->Set("running", Value::NewBool(false));
		this->running = false;

		this->InvokeOnExitCallback();

		// We must set complete to true after we call the callback, so
		// that it is only called once -- see this->Set(...)
		this->complete = true;

		this->parent->Terminated(this);
	}

	void Process::InvokeOnExitCallback()
	{
		SharedValue sv = this->Get("onexit");
		if (!sv->IsMethod())
		{
			return;
		}

		ValueList args(Value::NewInt(this->exitCode));
		SharedKMethod callback = sv->ToMethod();
		this->parent->GetHost()->InvokeMethodOnMainThread(
			callback, args, false);
	}

	void Process::InvokeOnReadCallback(bool isStdError)
	{
		Poco::ScopedLock<Poco::Mutex> lock(outputBufferMutex);
		SharedValue sv = this->Get("onread");
		if (!sv->IsMethod())
		{
			return;
		}

		std::string output;
		if (isStdError)
		{
			output = stdErrorBuffer.str();
			stdErrorBuffer.str("");
		}
		else
		{
			output = stdOutBuffer.str();
			stdOutBuffer.str("");
		}

		if (!output.empty())
		{
			ValueList args(
				Value::NewString(output),
				Value::NewBool(isStdError));
			SharedKMethod callback = sv->ToMethod();
			this->parent->GetHost()->InvokeMethodOnMainThread(
				callback, args, false);
		}
	}

	void Process::ReadStdOut()
	{
		while (this->running)
		{
			Poco::ScopedLock<Poco::Mutex> lock(outputBufferMutex);
			SharedValue result = Value::NewUndefined();
			this->out->Read(ValueList(), result);

			if (result->IsString())
			{
				stdOutBuffer << result->ToString();
				this->InvokeOnReadCallback(false);
			}
		}
	}

	void Process::ReadStdError()
	{
		while (this->running)
		{
			Poco::ScopedLock<Poco::Mutex> lock(outputBufferMutex);
			SharedValue result = Value::NewUndefined();
			this->err->Read(ValueList(), result);

			if (result->IsString())
			{
				stdErrorBuffer << result->ToString();
				this->InvokeOnReadCallback(true);
			}
		}
	}

	void Process::Terminate(const ValueList& args, SharedValue result)
	{
		Terminate();
	}

	void Process::Terminate()
	{
		KR_DUMP_LOCATION
		if (running)
		{
			this->running = false;
#ifdef OS_WIN32
			// win32 needs a kill to terminate process
			Poco::Process::kill(this->pid);
#else
			// this sends a more graceful SIGINT instead of SIGKILL
			// which is important for programs that manage child processes
			// and handle their own signals
			Poco::Process::requestTermination(this->pid);
#endif
			this->Set("running", Value::NewBool(false));
			this->parent->Terminated(this);
		}
	}

	void Process::Set(const char *name, SharedValue value)
	{
		KR_DUMP_LOCATION
		// We need to check the previous value of certain incomming values
		// *before* we actually do the Set(...) on this object.
		Poco::ScopedLock<Poco::Mutex> lock(outputBufferMutex);
		bool flushOnRead =
			(!strcmp("onread", name)) && (!this->Get("onread")->IsMethod());
		bool flushOnExit =
			(!strcmp("onexit", name)) && (!this->Get("onexit")->IsMethod());

		StaticBoundObject::Set(name, value);

		if (flushOnRead)
		{
			// If we had no previous onread callback flush our output
			// buffers, so that onread will be called even when it is
			// attached after a process finishes executing.
			this->InvokeOnReadCallback(false);
			this->InvokeOnReadCallback(true);
		}

		// this->complete is the signal that monitor thread has already
		// attempted to call the onexit callback. If it's false, the monitor
		// thread will take care of calling it.
		if (flushOnExit && this->complete)
		{
			this->InvokeOnExitCallback();
		}
	}
}

<commit_msg>Removed some debug output<commit_after>/**
 * Appcelerator Titanium - licensed under the Apache Public License 2
 * see LICENSE in the root folder for details on the license.
 * Copyright (c) 2009 Appcelerator, Inc. All Rights Reserved.
 */
#include <vector>
#include "process.h"
#include "pipe.h"
#include <Poco/Path.h>
#include <Poco/RunnableAdapter.h>
#include <Poco/ScopedLock.h>

using Poco::RunnableAdapter;

namespace ti
{
	Process::Process(ProcessBinding* parent, std::string& cmd, std::vector<std::string>& args) :
		running(false),
		complete(false),
		pid(-1),
		exitCode(-1),
		errp(0),
		outp(0),
		inp(0),
		logger(Logger::Get("Process.Process"))
	{
		this->parent = parent;
		this->errp = new Poco::Pipe();
		this->outp = new Poco::Pipe();
		this->inp = new Poco::Pipe();

		logger->Debug("Running process %s", cmd.c_str());
		try
		{
			this->arguments = args;
			this->command = cmd;
		}
		catch (std::exception &e)
		{
			throw ValueException::FromString(e.what());
		}

		/**
		 * @tiapi(property=True,type=string,name=Process.Process.command,version=0.2)
		 * @tiapi The command used for the Process object
		 */
		this->SetString("command", cmd);

		/**
		 * @tiapi(property=True,type=integer,name=Process.Process.pid,version=0.2)
		 * @tiapi The process id of the Process object
		 */
		this->SetNull("pid");

		/**
		 * @tiapi(property=True,type=boolean,name=Process.Process.running,version=0.2)
		 * @tiapi The running status of the Process object
		 */
		this->SetBool("running", false);

		/**
		 * @tiapi(property=True,type=object,name=Process.Process.err,version=0.2)
		 * @tiapi The Pipe object of the error stream
		 */
		this->err = new Pipe(new Poco::PipeInputStream(*errp));
		this->shared_error = new SharedKObject(this->err);
		this->SetObject("err", *shared_error);

		/**
		 * @tiapi(property=True,type=object,name=Process.Process.out,version=0.2)
		 * @tiapi The Pipe object of the output stream
		 */
		this->out = new Pipe(new Poco::PipeInputStream(*outp));
		this->shared_output = new SharedKObject(this->out);
		this->SetObject("out", *shared_output);

		/**
		 * @tiapi(property=True,type=object,name=Process.Process.in,version=0.2)
		 * @tiapi The Pipe object of the input stream
		 */
		this->in = new Pipe(new Poco::PipeOutputStream(*inp));
		this->shared_input = new SharedKObject(this->in);
		this->SetObject("in", *shared_input);

		/**
		 * @tiapi(method=True,name=Process.Process.terminate,version=0.2)
		 * @tiapi Terminates a running process
		 */
		this->SetMethod("terminate", &Process::Terminate);

		/**
		 * @tiapi(property=True,type=integer,name=Process.Process.exitCode,version=0.4)
		 * @tiapi The exit code or null if not yet exited
		 */
		this->SetNull("exitCode");

		/**
		 * @tiapi(property=True,type=method,name=Process.Process.onread,since=0.4)
		 * @tiapi The function handler to call when sys out is read
		 */
		this->SetNull("onread");

		/**
		 * @tiapi(property=True,type=method,name=Process.Process.onexit,since=0.4)
		 * @tiapi The function handler to call when the process exits
		 */
		this->SetNull("onexit");

		// setup threads which can read output and also monitor the exit
		this->monitorAdapter = new RunnableAdapter<Process>(*this, &Process::Monitor);
		this->exitMonitorThread.start(*monitorAdapter);
	}

	Process::~Process()
	{
		Terminate();

		if (this->exitMonitorThread.isRunning())
		{
			try
			{
				this->exitMonitorThread.join();
			}
			catch (Poco::Exception& e)
			{
				logger->Error(
					"Exception while try to join with exit monitor thread: %s",
					e.displayText().c_str());
			}
		}

		if (this->stdOutThread.isRunning())
		{
			try
			{
				this->stdOutThread.join();
			}
			catch (Poco::Exception& e)
			{
				logger->Error(
					"Exception while try to join with stdout thread: %s",
					e.displayText().c_str());
			}
		}

		if (this->stdErrorThread.isRunning())
		{
			try
			{
				this->stdErrorThread.join();
			}
			catch (Poco::Exception& e)
			{
				logger->Error(
					"Exception while try to join with stderr thread: %s",
					e.displayText().c_str());
			}
		}

		delete monitorAdapter;
		delete stdOutAdapter;
		delete stdErrorAdapter;
		delete shared_output;
		delete shared_input;
		delete shared_error;
	}

	void Process::StartReadThreads()
	{
		this->logger->Debug("Starting output handler threads...");
		if (!stdOutThread.isRunning())
		{
			this->stdOutAdapter = new RunnableAdapter<Process>(*this, &Process::ReadStdOut);
			stdOutThread.start(*stdOutAdapter);
		}

		if (!stdErrorThread.isRunning())
		{
			RunnableAdapter<Process> adapter(*this, &Process::ReadStdError);
			this->stdErrorAdapter = new RunnableAdapter<Process>(*this, &Process::ReadStdError);
			stdErrorThread.start(*stdErrorAdapter);
		}
	}

	void Process::Monitor()
	{
		this->Set("running", Value::NewBool(true));
		this->running = true;

		try
		{
			Poco::ProcessHandle ph = Poco::Process::launch(
				this->command, this->arguments,
				this->inp, this->outp, this->errp);
			this->StartReadThreads();
			this->pid = (int) ph.id();
			this->Set("pid", Value::NewInt(this->pid));

			this->exitCode = ph.wait();
			this->Set("exitCode", Value::NewInt(this->exitCode));
			logger->Debug("%s exited with return code %i",
				this->command.c_str(), this->exitCode);
		}
		catch (Poco::SystemException &se)
		{
			logger->Error("System Exception starting: %s, message: %s",
				this->command.c_str(), se.what());
		}
		catch (std::exception &e)
		{
			logger->Error("Exception starting: %s, message: %s",
				this->command.c_str(), e.what());
		}

		this->Set("running", Value::NewBool(false));
		this->running = false;

		this->InvokeOnExitCallback();

		// We must set complete to true after we call the callback, so
		// that it is only called once -- see this->Set(...)
		this->complete = true;

		this->parent->Terminated(this);
	}

	void Process::InvokeOnExitCallback()
	{
		SharedValue sv = this->Get("onexit");
		if (!sv->IsMethod())
		{
			return;
		}

		ValueList args(Value::NewInt(this->exitCode));
		SharedKMethod callback = sv->ToMethod();
		this->parent->GetHost()->InvokeMethodOnMainThread(
			callback, args, false);
	}

	void Process::InvokeOnReadCallback(bool isStdError)
	{
		Poco::ScopedLock<Poco::Mutex> lock(outputBufferMutex);
		SharedValue sv = this->Get("onread");
		if (!sv->IsMethod())
		{
			return;
		}

		std::string output;
		if (isStdError)
		{
			output = stdErrorBuffer.str();
			stdErrorBuffer.str("");
		}
		else
		{
			output = stdOutBuffer.str();
			stdOutBuffer.str("");
		}

		if (!output.empty())
		{
			ValueList args(
				Value::NewString(output),
				Value::NewBool(isStdError));
			SharedKMethod callback = sv->ToMethod();
			this->parent->GetHost()->InvokeMethodOnMainThread(
				callback, args, false);
		}
	}

	void Process::ReadStdOut()
	{
		while (this->running)
		{
			Poco::ScopedLock<Poco::Mutex> lock(outputBufferMutex);
			SharedValue result = Value::NewUndefined();
			this->out->Read(ValueList(), result);

			if (result->IsString())
			{
				stdOutBuffer << result->ToString();
				this->InvokeOnReadCallback(false);
			}
		}
	}

	void Process::ReadStdError()
	{
		while (this->running)
		{
			Poco::ScopedLock<Poco::Mutex> lock(outputBufferMutex);
			SharedValue result = Value::NewUndefined();
			this->err->Read(ValueList(), result);

			if (result->IsString())
			{
				stdErrorBuffer << result->ToString();
				this->InvokeOnReadCallback(true);
			}
		}
	}

	void Process::Terminate(const ValueList& args, SharedValue result)
	{
		Terminate();
	}

	void Process::Terminate()
	{
		if (running)
		{
			this->running = false;
#ifdef OS_WIN32
			// win32 needs a kill to terminate process
			Poco::Process::kill(this->pid);
#else
			// this sends a more graceful SIGINT instead of SIGKILL
			// which is important for programs that manage child processes
			// and handle their own signals
			Poco::Process::requestTermination(this->pid);
#endif
			this->Set("running", Value::NewBool(false));
			this->parent->Terminated(this);
		}
	}

	void Process::Set(const char *name, SharedValue value)
	{
		// We need to check the previous value of certain incomming values
		// *before* we actually do the Set(...) on this object.
		Poco::ScopedLock<Poco::Mutex> lock(outputBufferMutex);
		bool flushOnRead =
			(!strcmp("onread", name)) && (!this->Get("onread")->IsMethod());
		bool flushOnExit =
			(!strcmp("onexit", name)) && (!this->Get("onexit")->IsMethod());

		StaticBoundObject::Set(name, value);

		if (flushOnRead)
		{
			// If we had no previous onread callback flush our output
			// buffers, so that onread will be called even when it is
			// attached after a process finishes executing.
			this->InvokeOnReadCallback(false);
			this->InvokeOnReadCallback(true);
		}

		// this->complete is the signal that monitor thread has already
		// attempted to call the onexit callback. If it's false, the monitor
		// thread will take care of calling it.
		if (flushOnExit && this->complete)
		{
			this->InvokeOnExitCallback();
		}
	}
}

<|endoftext|>


<commit_before>/*
 * Copyright 2014 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <thrift/lib/cpp2/async/RequestChannel.h>

#include <memory>
#include <folly/Memory.h>
#include <folly/io/async/EventBase.h>
#include <folly/io/async/AsyncServerSocket.h>
#include <thrift/lib/cpp/async/TAsyncSocket.h>
#include <thrift/lib/cpp2/async/HeaderClientChannel.h>
#include <thrift/lib/cpp2/util/ScopedServerInterfaceThread.h>
#include <thrift/lib/cpp2/test/gen-cpp2/TestService.h>

#include <gtest/gtest.h>
#include <gmock/gmock.h>

using namespace std;
using namespace folly;
using namespace apache::thrift;
using namespace apache::thrift::async;
using namespace apache::thrift::test::cpp2;
using namespace apache::thrift::transport;
using namespace testing;

using CSR = ClientReceiveState;

//  This binds to an ephemeral port but does not listen.
//  We therefore know at least one port which is guaranteed not to be listening.
//  Useful for testing server-down cases.
class PortHolder {
 public:
  PortHolder() {
    th_ = thread([&]{ eb_.loopForever(); });
    eb_.waitUntilRunning();
    sock_ = AsyncServerSocket::newSocket(&eb_);
    sock_->bind(0);
  }
  ~PortHolder() {
    eb_.terminateLoopSoon();
    th_.join();
  }
  folly::SocketAddress getAddress() {
    folly::SocketAddress ret;
    sock_->getAddress(&ret);
    return ret;
  }
 private:
  EventBase eb_;
  thread th_;
  shared_ptr<AsyncServerSocket> sock_;
};

class TestServiceServerMock : public TestServiceSvIf {
 public:
  MOCK_METHOD1(noResponse, void(int64_t));
};

class FunctionSendCallbackTest : public Test {
 public:
  unique_ptr<TestServiceAsyncClient> getClient(
      const folly::SocketAddress& addr) {
    return make_unique<TestServiceAsyncClient>(
      HeaderClientChannel::newChannel(TAsyncSocket::newSocket(&eb, addr)));
  }
  void sendOnewayMessage(
      const folly::SocketAddress& addr,
      function<void(ClientReceiveState&&)> cb) {
    auto client = getClient(addr);
    client->noResponse(make_unique<FunctionSendCallback>(move(cb)),
                       68 /* without loss of generality */);
    eb.loop();
  }
  EventBase eb;
};

TEST_F(FunctionSendCallbackTest, with_missing_server_fails) {
  PortHolder ph;
  exception_wrapper exn;
  sendOnewayMessage(ph.getAddress(), [&](CSR&& state) {
      exn = state.exceptionWrapper();
  });
  EXPECT_TRUE(bool(exn));
  auto err = "transport is closed in write()";
  EXPECT_NE(string::npos, exn.what().find(err));
}

TEST_F(FunctionSendCallbackTest, with_throwing_server_passes) {
  auto si = make_shared<TestServiceServerMock>();
  ScopedServerInterfaceThread ssit(si);
  Baton<> done;
  EXPECT_CALL(*si, noResponse(_)).WillOnce(DoAll(
        Invoke([&](int64_t _) { done.post(); }),
        Throw(runtime_error("hi"))));
  exception_wrapper exn = make_exception_wrapper<runtime_error>("lo");
  sendOnewayMessage(ssit.getAddress(), [&](CSR&& state) {
      exn = state.exceptionWrapper();
  });
  done.timed_wait(chrono::steady_clock::now() + chrono::milliseconds(50));
  EXPECT_FALSE(exn);
}
<commit_msg>ScopedBoundPort<commit_after>/*
 * Copyright 2014 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <thrift/lib/cpp2/async/RequestChannel.h>

#include <memory>
#include <folly/Memory.h>
#include <folly/io/async/EventBase.h>
#include <folly/io/async/test/ScopedBoundPort.h>
#include <thrift/lib/cpp/async/TAsyncSocket.h>
#include <thrift/lib/cpp2/async/HeaderClientChannel.h>
#include <thrift/lib/cpp2/util/ScopedServerInterfaceThread.h>
#include <thrift/lib/cpp2/test/gen-cpp2/TestService.h>

#include <gtest/gtest.h>
#include <gmock/gmock.h>

using namespace std;
using namespace folly;
using namespace apache::thrift;
using namespace apache::thrift::async;
using namespace apache::thrift::test::cpp2;
using namespace apache::thrift::transport;
using namespace testing;

using CSR = ClientReceiveState;

class TestServiceServerMock : public TestServiceSvIf {
 public:
  MOCK_METHOD1(noResponse, void(int64_t));
};

class FunctionSendCallbackTest : public Test {
 public:
  unique_ptr<TestServiceAsyncClient> getClient(
      const folly::SocketAddress& addr) {
    return make_unique<TestServiceAsyncClient>(
      HeaderClientChannel::newChannel(TAsyncSocket::newSocket(&eb, addr)));
  }
  void sendOnewayMessage(
      const folly::SocketAddress& addr,
      function<void(ClientReceiveState&&)> cb) {
    auto client = getClient(addr);
    client->noResponse(make_unique<FunctionSendCallback>(move(cb)),
                       68 /* without loss of generality */);
    eb.loop();
  }
  EventBase eb;
};

TEST_F(FunctionSendCallbackTest, with_missing_server_fails) {
  ScopedBoundPort bound;
  exception_wrapper exn;
  sendOnewayMessage(bound.getAddress(), [&](CSR&& state) {
      exn = state.exceptionWrapper();
  });
  EXPECT_TRUE(bool(exn));
  auto err = "transport is closed in write()";
  EXPECT_NE(string::npos, exn.what().find(err));
}

TEST_F(FunctionSendCallbackTest, with_throwing_server_passes) {
  auto si = make_shared<TestServiceServerMock>();
  ScopedServerInterfaceThread ssit(si);
  Baton<> done;
  EXPECT_CALL(*si, noResponse(_)).WillOnce(DoAll(
        Invoke([&](int64_t _) { done.post(); }),
        Throw(runtime_error("hi"))));
  exception_wrapper exn = make_exception_wrapper<runtime_error>("lo");
  sendOnewayMessage(ssit.getAddress(), [&](CSR&& state) {
      exn = state.exceptionWrapper();
  });
  done.timed_wait(chrono::steady_clock::now() + chrono::milliseconds(50));
  EXPECT_FALSE(exn);
}
<|endoftext|>