<commit_before>/*******************************************************************************
 * thrill/api/groupby.hpp
 *
 * DIANode for a groupby operation. Performs the actual groupby operation
 *
 * Part of Project Thrill.
 *
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#pragma once
#ifndef THRILL_API_GROUPBY_HEADER
#define THRILL_API_GROUPBY_HEADER

#include <thrill/api/dia.hpp>
#include <thrill/api/dop_node.hpp>
#include <thrill/common/functional.hpp>
#include <thrill/common/logger.hpp>
#include <thrill/core/iterator_wrapper.hpp>
#include <thrill/core/stxxl_multiway_merge.hpp>

#include <functional>
#include <string>
#include <type_traits>
#include <typeinfo>
#include <utility>
#include <vector>

namespace thrill {
namespace api {

template <typename ValueTypeIn>
class GroupByIterator {
public:
    GroupByIterator(data::File::Reader &r) : r_(r) {}

    bool HasNext() {
        return r_.HasNext();
    }

    ValueTypeIn Next() {
        return r_.template Next<ValueTypeIn>();
    }
private:
    data::File::Reader &r_;
};

template <typename ValueTypeIn, typename ParentDIARef,
          typename KeyExtractor, typename GroupFunction, typename HashFunction>
class GroupByNode : public DOpNode<ValueTypeIn>
{
    static const bool debug = false;
    using Super = DOpNode<ValueTypeIn>;
    using Key = typename common::FunctionTraits<KeyExtractor>::result_type;
    using ValueTypeOut = typename common::FunctionTraits<GroupFunction>::result_type;
    using ReduceArg = typename common::FunctionTraits<GroupFunction>
                      ::template arg<0>;
    using KeyValuePair = typename std::pair<Key, ValueTypeIn>;

    struct ValueComparator
    {
        ValueComparator(const GroupByNode& info) : info_(info) { }
        const GroupByNode& info_;

        bool operator () (const ValueTypeIn& i,
                          const ValueTypeIn& j) {
            auto i_cmp = info_.hash_function_(info_.key_extractor_(i));
            auto j_cmp = info_.hash_function_(info_.key_extractor_(j));
            return (i_cmp < j_cmp);
        }
    };

    using Super::context_;
    using Super::result_file_;

public:
    /*!
     * Constructor for a GroupByNode. Sets the DataManager, parent, stack,
     * key_extractor and reduce_function.
     *
     * \param parent Parent DIARef.
     * and this node
     * \param key_extractor Key extractor function
     * \param reduce_function Reduce function
     */
    GroupByNode(const ParentDIARef& parent,
                KeyExtractor key_extractor,
                GroupFunction groupby_function,
                StatsNode* stats_node,
                const HashFunction& hash_function = HashFunction())
        : DOpNode<ValueTypeIn>(parent.ctx(), { parent.node() }, "GroupBy", stats_node),
          key_extractor_(key_extractor),
          groupby_function_(groupby_function),
          hash_function_(hash_function),
          channel_(parent.ctx().GetNewChannel()),
          emitter_(channel_->OpenWriters())
    {
        // Hook PreOp
        auto pre_op_fn = [=](const ValueTypeIn& input) {
                             PreOp(input);
                         };
        // close the function stack with our pre op and register it at
        // parent node for output
        auto lop_chain = parent.stack().push(pre_op_fn).emit();
        parent.node()->RegisterChild(lop_chain, this->type());
        channel_->OnClose([this]() {
                              this->WriteChannelStats(this->channel_);
                          });
    }

    //! Virtual destructor for a GroupByNode.
    virtual ~GroupByNode() { }

    /*!
     * Actually executes the reduce operation. Uses the member functions PreOp,
     * MainOp and PostOp.
     */
    void Execute() override {
        MainOp();
    }

    void ProcessGroup(data::File& f) {
        auto r = f.GetReader();
        std::vector<std::function<void(const ValueTypeIn&)> > cbs;
        DIANode<ValueTypeIn>::callback_functions(cbs);
    }

    void PushData() override {
        // OMG THIS IS SO HACKY. THIS MUST BE POSSIBLE MORE ELEGANTLY
        // split sorted files to multiple files for each key
        std::vector<data::File> user_files;
        auto r = sorted_elems_.GetReader();
        if (r.HasNext()) {
            ValueTypeIn v1 = r.template Next<ValueTypeIn>();
            Key k1 = key_extractor_(v1);
            bool inserted = false;
            while (r.HasNext()) {
                user_files.push_back(data::File());
                {
                    auto w = user_files.back().GetWriter();
                    w(v1);
                    LOG << "Host " << context_.host_rank() << " added " << v1;
                    inserted = true;

                    while (inserted && r.HasNext()) {
                        ValueTypeIn v2 = r.template Next<ValueTypeIn>();
                        Key k2 = key_extractor_(v2);
                        inserted = false;
                        if (k2 == k1) {
                            w(v2);
                            LOG << "Host " << context_.host_rank() << " added " << v2;
                            inserted = true;
                        }
                        else {
                            v1 = v2;
                            k1 = k2;
                        }
                    }
                }
            }
        }

        {
        auto result_writer = result_.GetWriter();
            // call user function
            for (auto t : user_files) {
                auto r = t.GetReader();
                data_.push_back(
                    groupby_function_(GroupByIterator<ValueTypeIn>(r)));
            }
        }

        // push data to callback functions
        for (size_t i = 0; i < data_.size(); i++) {
            for (auto func : DIANode<ValueTypeIn>::callbacks_) {
                LOG << "Host " << context_.host_rank() << " grouped to value " << data_[i];
                func(data_[i]);
            }
        }
    }

    void Dispose() override { }

    /*!
     * Produces a function stack, which only contains the PostOp function.
     * \return PostOp function stack
     */
    auto ProduceStack() {
        return FunctionStack<ValueTypeIn>();
    }

    /*!
     * Returns "[GroupByNode]" and its id as a string.
     * \return "[GroupByNode]"
     */
    std::string ToString() override {
        return "[GroupByNode] Id: " + result_file_.ToString();
    }

private:
    KeyExtractor key_extractor_;
    GroupFunction groupby_function_;
    HashFunction hash_function_;

    data::ChannelPtr channel_;
    std::vector<data::BlockWriter> emitter_;
    std::vector<data::File> files_;
    data::File sorted_elems_;
    std::vector<ValueTypeOut> data_;
    data::File result_;

    /*
     * Send all elements to their designated PEs
     */
    void PreOp(const ValueTypeIn& v) {
        Key k = key_extractor_(v);
        auto recipient = hash_function_(k) % emitter_.size();
        emitter_[recipient](v);
    }

    /*
     * Sort and store elements in a file
     */
    void FlushVectorToFile(std::vector<ValueTypeIn>& v) {
        // sort run and sort to file
        std::sort(v.begin(), v.end(), ValueComparator(*this));
        data::File f;
        {
            auto w = f.GetWriter();
            for (auto& e : v) {
                w(e);
            }
        }

        files_.push_back(f);
    }

    //! Receive elements from other workers.
    auto MainOp() {
        using Iterator = thrill::core::StxxlFileWrapper<ValueTypeIn>;
        using OIterator = thrill::core::StxxlFileOutputWrapper<int>;
        using File = data::File;
        using Reader = File::Reader;
        using Writer = File::Writer;

        LOG << ToString() << " running main op";

        const size_t FIXED_VECTOR_SIZE = 99999;
        std::vector<ValueTypeIn> incoming;
        incoming.reserve(FIXED_VECTOR_SIZE);

        // close all emitters
        for (auto& e : emitter_) {
            e.Close();
        }

        size_t totalsize = 0;

        // get incoming elements
        auto reader = channel_->OpenReader();
        while (reader.HasNext()) {
            // if vector is full save to disk
            if (incoming.size() == FIXED_VECTOR_SIZE) {
                FlushVectorToFile(incoming);
                incoming.clear();
            }
            // store incoming element
            auto elem = reader.template Next<ValueTypeIn>();
            incoming.push_back(elem);
            LOG << "Host " << context_.host_rank() << " received " << elem << " with key " << key_extractor_(incoming.back());
            ++totalsize;
        }
        FlushVectorToFile(incoming);

        // if there's only one run, store it
        if (files_.size() == 1) {
            sorted_elems_ = files_[0];
        } // otherwise sort all runs using multiway merge
        else {
            std::vector<std::pair<Iterator, Iterator> > seq;
            seq.reserve(files_.size());
            for (std::size_t t = 0; t < files_.size(); ++t) {
                auto reader = std::make_shared<Reader>(files_[t].GetReader());
                Iterator s = Iterator(&files_[t], reader, 0, true);
                Iterator e = Iterator(&files_[t], reader, files_[t].NumItems(), false);
                seq.push_back(std::make_pair(s, e));
            }

            {
                OIterator oiter(std::make_shared<Writer>(sorted_elems_.GetWriter()));

                stxxl::parallel::sequential_file_multiway_merge<true, false>(
                    std::begin(seq),
                    std::end(seq),
                    oiter,
                    totalsize,
                    ValueComparator(*this));
            }
        }
    }
};

/******************************************************************************/

template <typename ValueTypeIn, typename Stack>
template <typename KeyExtractor, typename GroupFunction, typename HashFunction>
auto DIARef<ValueTypeIn, Stack>::GroupBy(
    const KeyExtractor &key_extractor,
    const GroupFunction &groupby_function) const {

    using DOpResult
              = typename common::FunctionTraits<GroupFunction>::result_type;

    // TODO(cn) find correct assertions for input paarms

    StatsNode* stats_node = AddChildStatsNode("GroupBy", NodeType::DOP);
    using GroupByResultNode
              = GroupByNode<DOpResult, DIARef, KeyExtractor,
                            GroupFunction, HashFunction>;
    auto shared_node
        = std::make_shared<GroupByResultNode>(*this,
                                              key_extractor,
                                              groupby_function,
                                              stats_node);

    auto groupby_stack = shared_node->ProduceStack();

    return DIARef<DOpResult, decltype(groupby_stack)>(
        shared_node,
        groupby_stack,
        { stats_node });
}

} // namespace api
} // namespace thrill

#endif // !THRILL_API_GROUPBY_HEADER

/******************************************************************************/
<commit_msg>add more const and other cleanups<commit_after>/*******************************************************************************
 * thrill/api/groupby.hpp
 *
 * DIANode for a groupby operation. Performs the actual groupby operation
 *
 * Part of Project Thrill.
 *
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#pragma once
#ifndef THRILL_API_GROUPBY_HEADER
#define THRILL_API_GROUPBY_HEADER

#include <thrill/api/dia.hpp>
#include <thrill/api/dop_node.hpp>
#include <thrill/common/functional.hpp>
#include <thrill/common/logger.hpp>
#include <thrill/core/iterator_wrapper.hpp>
#include <thrill/core/stxxl_multiway_merge.hpp>

#include <functional>
#include <string>
#include <type_traits>
#include <typeinfo>
#include <utility>
#include <vector>

namespace thrill {
namespace api {

template <typename ValueTypeIn>
class GroupByIterator {
public:
    GroupByIterator(data::File::Reader &r) : r_(r) {}

    bool HasNext() {
        return r_.HasNext();
    }

    ValueTypeIn Next() {
        return r_.template Next<ValueTypeIn>();
    }
private:
    data::File::Reader &r_;
};

template <typename ValueTypeIn, typename ParentDIARef,
          typename KeyExtractor, typename GroupFunction, typename HashFunction>
class GroupByNode : public DOpNode<ValueTypeIn>
{
    static const bool debug = false;
    using Super = DOpNode<ValueTypeIn>;
    using Key = typename common::FunctionTraits<KeyExtractor>::result_type;
    using ValueTypeOut = typename common::FunctionTraits<GroupFunction>::result_type;
    using ReduceArg = typename common::FunctionTraits<GroupFunction>
                      ::template arg<0>;
    using KeyValuePair = typename std::pair<Key, ValueTypeIn>;

    struct ValueComparator
    {
        ValueComparator(const GroupByNode& info) : info_(info) { }
        const GroupByNode& info_;

        bool operator () (const ValueTypeIn& i,
                          const ValueTypeIn& j) {
            auto i_cmp = info_.hash_function_(info_.key_extractor_(i));
            auto j_cmp = info_.hash_function_(info_.key_extractor_(j));
            return (i_cmp < j_cmp);
        }
    };

    using Super::context_;
    using Super::result_file_;

public:
    /*!
     * Constructor for a GroupByNode. Sets the DataManager, parent, stack,
     * key_extractor and reduce_function.
     *
     * \param parent Parent DIARef.
     * and this node
     * \param key_extractor Key extractor function
     * \param reduce_function Reduce function
     */
    GroupByNode(const ParentDIARef& parent,
                KeyExtractor key_extractor,
                GroupFunction groupby_function,
                StatsNode* stats_node,
                const HashFunction& hash_function = HashFunction())
        : DOpNode<ValueTypeIn>(parent.ctx(), { parent.node() }, "GroupBy", stats_node),
          key_extractor_(key_extractor),
          groupby_function_(groupby_function),
          hash_function_(hash_function),
          channel_(parent.ctx().GetNewChannel()),
          emitter_(channel_->OpenWriters())
    {
        // Hook PreOp
        auto pre_op_fn = [=](const ValueTypeIn& input) {
                             PreOp(input);
                         };
        // close the function stack with our pre op and register it at
        // parent node for output
        auto lop_chain = parent.stack().push(pre_op_fn).emit();
        parent.node()->RegisterChild(lop_chain, this->type());
        channel_->OnClose([this]() {
                              this->WriteChannelStats(this->channel_);
                          });
    }

    //! Virtual destructor for a GroupByNode.
    virtual ~GroupByNode() { }

    /*!
     * Actually executes the reduce operation. Uses the member functions PreOp,
     * MainOp and PostOp.
     */
    void Execute() override {
        MainOp();
    }

    void ProcessGroup(data::File& f) {
        auto r = f.GetReader();
        std::vector<std::function<void(const ValueTypeIn&)> > cbs;
        DIANode<ValueTypeIn>::callback_functions(cbs);
    }

    void PushData() override {
        // OMG THIS IS SO HACKY. THIS MUST BE POSSIBLE MORE ELEGANTLY
        // split sorted files to multiple files for each key
        std::vector<data::File> user_files;
        auto r = sorted_elems_.GetReader();
        if (r.HasNext()) {
            ValueTypeIn v1 = r.template Next<ValueTypeIn>();
            Key k1 = key_extractor_(v1);
            bool inserted = false;
            while (r.HasNext()) {
                user_files.push_back(data::File());
                {
                    auto w = user_files.back().GetWriter();
                    w(v1);
                    LOG << "Host " << context_.host_rank() << " added " << v1;
                    inserted = true;

                    while (inserted && r.HasNext()) {
                        ValueTypeIn v2 = r.template Next<ValueTypeIn>();
                        Key k2 = key_extractor_(v2);
                        inserted = false;
                        if (k2 == k1) {
                            w(v2);
                            LOG << "Host " << context_.host_rank() << " added " << v2;
                            inserted = true;
                        }
                        else {
                            v1 = v2;
                            k1 = k2;
                        }
                    }
                }
            }
        }

        // call user function
        for (auto t : user_files) {
            auto r = t.GetReader();
            const auto res = groupby_function_(GroupByIterator<ValueTypeIn>(r));
            // push result to callback functions
            for (auto func : DIANode<ValueTypeIn>::callbacks_) {
                LOG << "Host " << context_.host_rank() << " grouped to value " << res;
                func(res);
            }
        }
    }

    void Dispose() override { }

    /*!
     * Produces a function stack, which only contains the PostOp function.
     * \return PostOp function stack
     */
    auto ProduceStack() {
        return FunctionStack<ValueTypeIn>();
    }

    /*!
     * Returns "[GroupByNode]" and its id as a string.
     * \return "[GroupByNode]"
     */
    std::string ToString() override {
        return "[GroupByNode] Id: " + result_file_.ToString();
    }

private:
    KeyExtractor key_extractor_;
    GroupFunction groupby_function_;
    HashFunction hash_function_;

    data::ChannelPtr channel_;
    std::vector<data::BlockWriter> emitter_;
    std::vector<data::File> files_;
    data::File sorted_elems_;

    /*
     * Send all elements to their designated PEs
     */
    void PreOp(const ValueTypeIn& v) {
        const Key k = key_extractor_(v);
        const auto recipient = hash_function_(k) % emitter_.size();
        emitter_[recipient](v);
    }

    /*
     * Sort and store elements in a file
     */
    void FlushVectorToFile(std::vector<ValueTypeIn>& v) {
        // sort run and sort to file
        std::sort(v.begin(), v.end(), ValueComparator(*this));
        data::File f;
        {
            auto w = f.GetWriter();
            for (const auto& e : v) {
                w(e);
            }
        }

        files_.push_back(f);
    }

    //! Receive elements from other workers.
    auto MainOp() {
        using Iterator = thrill::core::StxxlFileWrapper<ValueTypeIn>;
        using OIterator = thrill::core::StxxlFileOutputWrapper<int>;
        using File = data::File;
        using Reader = File::Reader;
        using Writer = File::Writer;

        LOG << ToString() << " running group by main op";

        const std::size_t FIXED_VECTOR_SIZE = 99999;
        std::vector<ValueTypeIn> incoming;
        incoming.reserve(FIXED_VECTOR_SIZE);

        // close all emitters
        for (auto& e : emitter_) {
            e.Close();
        }

        std::size_t totalsize = 0;

        // get incoming elements
        auto reader = channel_->OpenReader();
        while (reader.HasNext()) {
            // if vector is full save to disk
            if (incoming.size() == FIXED_VECTOR_SIZE) {
                FlushVectorToFile(incoming);
                incoming.clear();
            }
            // store incoming element
            const auto elem = reader.template Next<ValueTypeIn>();
            incoming.push_back(elem);
            LOG << "Host " << context_.host_rank() << " received " << elem << " with key " << key_extractor_(incoming.back());
            ++totalsize;
        }
        FlushVectorToFile(incoming);

        const auto num_elems = files_.size();
        // if there's only one run, store it
        if (num_elems == 1) {
            sorted_elems_ = std::move(files_[0]);
        } // otherwise sort all runs using multiway merge
        else {
            std::vector<std::pair<Iterator, Iterator> > seq;
            seq.reserve(num_elems);
            for (std::size_t t = 0; t < num_elems; ++t) {
                auto reader = std::make_shared<Reader>(files_[t].GetReader());
                Iterator s = Iterator(&files_[t], reader, 0, true);
                Iterator e = Iterator(&files_[t], reader, files_[t].NumItems(), false);
                seq.push_back(std::make_pair(s, e));
            }

            {
                OIterator oiter(std::make_shared<Writer>(sorted_elems_.GetWriter()));

                stxxl::parallel::sequential_file_multiway_merge<true, false>(
                    std::begin(seq),
                    std::end(seq),
                    oiter,
                    totalsize,
                    ValueComparator(*this));
            }
        }
    }
};

/******************************************************************************/

template <typename ValueTypeIn, typename Stack>
template <typename KeyExtractor, typename GroupFunction, typename HashFunction>
auto DIARef<ValueTypeIn, Stack>::GroupBy(
    const KeyExtractor &key_extractor,
    const GroupFunction &groupby_function) const {

    using DOpResult
              = typename common::FunctionTraits<GroupFunction>::result_type;

    // TODO(cn) find correct assertions for input paarms

    StatsNode* stats_node = AddChildStatsNode("GroupBy", NodeType::DOP);
    using GroupByResultNode
              = GroupByNode<DOpResult, DIARef, KeyExtractor,
                            GroupFunction, HashFunction>;
    auto shared_node
        = std::make_shared<GroupByResultNode>(*this,
                                              key_extractor,
                                              groupby_function,
                                              stats_node);

    auto groupby_stack = shared_node->ProduceStack();

    return DIARef<DOpResult, decltype(groupby_stack)>(
        shared_node,
        groupby_stack,
        { stats_node });
}

} // namespace api
} // namespace thrill

#endif // !THRILL_API_GROUPBY_HEADER

/******************************************************************************/
<|endoftext|>


<commit_before>// Copyright (c) 2006-2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/gpu/gpu_info_collector.h"

namespace gpu_info_collector {

bool CollectGraphicsInfo(GPUInfo* gpu_info) {
  // TODO(rlp): complete this function
  // TODO(apatrick): this illustrates how can_lose_context will be implemented
  // on this platform in the future.
  // bool can_lose_context =
  //     gfx::GetGLImplementation() == gfx::kGLImplementationEGLGLES2;

  gpu_info->SetProgress(GPUInfo::kComplete);

  return true;
}

}  // namespace gpu_info_collector
<commit_msg>Collect GPU information (vendor id and device id) in Linux.<commit_after>// Copyright (c) 2006-2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/gpu/gpu_info_collector.h"

#include <dlfcn.h>
#include <vector>

#include "app/gfx/gl/gl_bindings.h"
#include "app/gfx/gl/gl_context.h"
#include "app/gfx/gl/gl_implementation.h"
#include "base/logging.h"
#include "base/scoped_ptr.h"
#include "base/string_util.h"

namespace {

// PciDevice and PciAccess are defined to access libpci functions.  Their
// members match the corresponding structures defined by libpci in size up to
// fields we may access.  For those members we don't use, their names are
// defined as "fieldX", etc., or, left out if they are declared after the
// members we care about in libpci.

struct PciDevice {
  PciDevice* next;

  uint16 field0;
  uint8 field1;
  uint8 field2;
  uint8 field3;
  int field4;

  uint16 vendor_id;
  uint16 device_id;
  uint16 device_class;
};

struct PciAccess {
  unsigned int field0;
  int field1;
  int field2;
  char* field3;
  int field4;
  int field5;
  unsigned int field6;
  int field7;

  void (*function0)();
  void (*function1)();
  void (*function2)();

  PciDevice* device_list;
};

// Define function types.
typedef PciAccess* (*FT_pci_alloc)();
typedef void (*FT_pci_init)(PciAccess*);
typedef void (*FT_pci_cleanup)(PciAccess*);
typedef void (*FT_pci_scan_bus)(PciAccess*);
typedef void (*FT_pci_scan_bus)(PciAccess*);
typedef int (*FT_pci_fill_info)(PciDevice*, int);
typedef char* (*FT_pci_lookup_name)(PciAccess*, char*, int, int, ...);

// This includes dynamically linked library handle and functions pointers from
// libpci.
struct PciInterface {
  void* lib_handle;

  FT_pci_alloc pci_alloc;
  FT_pci_init pci_init;
  FT_pci_cleanup pci_cleanup;
  FT_pci_scan_bus pci_scan_bus;
  FT_pci_fill_info pci_fill_info;
  FT_pci_lookup_name pci_lookup_name;
};

// This dynamically opens libpci and get function pointers we need.  Return
// NULL if library fails to open or any functions can not be located.
// Returned interface (if not NULL) should be deleted in FinalizeLibPci.
PciInterface* InitializeLibPci(const char* lib_name) {
  void* handle = dlopen(lib_name, RTLD_LAZY);
  if (handle == NULL) {
    LOG(ERROR) << "Fail to dlopen libpci";
    return NULL;
  }
  PciInterface* interface = new struct PciInterface;
  interface->lib_handle = handle;
  interface->pci_alloc = reinterpret_cast<FT_pci_alloc>(
      dlsym(handle, "pci_alloc"));
  interface->pci_init = reinterpret_cast<FT_pci_init>(
      dlsym(handle, "pci_init"));
  interface->pci_cleanup = reinterpret_cast<FT_pci_cleanup>(
      dlsym(handle, "pci_cleanup"));
  interface->pci_scan_bus = reinterpret_cast<FT_pci_scan_bus>(
      dlsym(handle, "pci_scan_bus"));
  interface->pci_fill_info = reinterpret_cast<FT_pci_fill_info>(
      dlsym(handle, "pci_fill_info"));
  interface->pci_lookup_name = reinterpret_cast<FT_pci_lookup_name>(
      dlsym(handle, "pci_lookup_name"));
  if (interface->pci_alloc == NULL ||
      interface->pci_init == NULL ||
      interface->pci_cleanup == NULL ||
      interface->pci_scan_bus == NULL ||
      interface->pci_fill_info == NULL ||
      interface->pci_lookup_name == NULL) {
    LOG(ERROR) << "Missing required function(s) from libpci";
    dlclose(handle);
    delete interface;
    return NULL;
  }
  return interface;
}

// This close the dynamically opened libpci and delete the interface.
void FinalizeLibPci(PciInterface*& interface) {
  DCHECK(interface != NULL && interface->lib_handle != NULL);
  dlclose(interface->lib_handle);
  delete interface;
  interface = NULL;
}

// This creates an offscreen GL context for gl queries.  Returned GLContext
// should be deleted in FinalizeGLContext.
gfx::GLContext* InitializeGLContext() {
  if (!gfx::GLContext::InitializeOneOff()) {
    LOG(ERROR) << "gfx::GLContext::InitializeOneOff() failed";
    return NULL;
  }
  gfx::GLContext* context = gfx::GLContext::CreateOffscreenGLContext(NULL);
  if (context == NULL) {
    LOG(ERROR) << "gfx::GLContext::CreateOffscreenGLContext(NULL) failed";
    return NULL;
  }
  if (!context->MakeCurrent()) {
    LOG(ERROR) << "gfx::GLContext::MakeCurrent() failed";
    context->Destroy();
    delete context;
    return NULL;
  }
  return context;
}

// This destroy and delete the GL context.
void FinalizeGLContext(gfx::GLContext*& context) {
  DCHECK(context != NULL);
  context->Destroy();
  delete context;
  context = NULL;
}

}  // namespace anonymous

namespace gpu_info_collector {

bool CollectGraphicsInfo(GPUInfo* gpu_info) {
  gfx::GLContext* context = InitializeGLContext();
  if (context == NULL)
    return false;

  // TODO(zmo): collect driver version, pixel shader version, vertex shader
  // version, and gl version.
  std::wstring driver_version = L"";
  uint32 pixel_shader_version = 0;
  uint32 vertex_shader_version = 0;
  uint32 gl_version = 0;
  bool can_lose_context =
      (gfx::GetGLImplementation() == gfx::kGLImplementationEGLGLES2);

  // TODO(zmo): be more flexible about library name.
  PciInterface* interface = InitializeLibPci("libpci.so.3");
  if (interface == NULL) {
    FinalizeGLContext(context);
    return false;
  }

  PciAccess* access = (interface->pci_alloc)();
  DCHECK(access != NULL);
  (interface->pci_init)(access);
  (interface->pci_scan_bus)(access);
  std::vector<PciDevice*> gpu_list;
  PciDevice* gpu_active = NULL;
  for (PciDevice* device = access->device_list;
       device != NULL; device = device->next) {
    (interface->pci_fill_info)(device, 33);  // Fill the IDs and class fields.
    if (device->device_class == 0x0300) {  // Device class is DISPLAY_VGA.
      gpu_list.push_back(device);
    }
  }
  if (gpu_list.size() == 1) {
    gpu_active = gpu_list[0];
  } else {
    // If more than one graphics card are identified, find the one that matches
    // gl VENDOR and RENDERER info.
    std::string gl_vendor_string =
        reinterpret_cast<const char*>(glGetString(GL_VENDOR));
    std::string gl_renderer_string =
        reinterpret_cast<const char*>(glGetString(GL_RENDERER));
    const int buffer_size = 255;
    scoped_array<char> buffer(new char[buffer_size]);
    std::vector<PciDevice*> candidates;
    for (size_t i = 0; i < gpu_list.size(); ++i) {
      PciDevice* gpu = gpu_list[i];
      // The current implementation of pci_lookup_name returns the same pointer
      // as the passed in upon success, and a different one (NULL or a pointer
      // to an error message) upon failure.
      if ((interface->pci_lookup_name)(access,
                                       buffer.get(),
                                       buffer_size,
                                       1,
                                       gpu->vendor_id) != buffer.get())
        continue;
      std::string vendor_string = buffer.get();
      const bool kCaseSensitive = false;
      if (!StartsWithASCII(gl_vendor_string, vendor_string, kCaseSensitive))
        continue;
      if ((interface->pci_lookup_name)(access,
                                       buffer.get(),
                                       buffer_size,
                                       2,
                                       gpu->vendor_id,
                                       gpu->device_id) != buffer.get())
        continue;
      std::string device_string = buffer.get();
      size_t begin = device_string.find_first_of('[');
      size_t end = device_string.find_last_of(']');
      if (begin != std::string::npos && end != std::string::npos &&
          begin < end) {
        device_string = device_string.substr(begin + 1, end - begin - 1);
      }
      if (StartsWithASCII(gl_renderer_string, device_string, kCaseSensitive)) {
        gpu_active = gpu;
        break;
      }
      // If a device's vendor matches gl VENDOR string, we want to consider the
      // possibility that libpci may not return the exact same name as gl
      // RENDERER string.
      candidates.push_back(gpu);
    }
    if (gpu_active == NULL && candidates.size() == 1)
      gpu_active = candidates[0];
  }
  if (gpu_active != NULL) {
    gpu_info->SetGraphicsInfo(gpu_active->vendor_id,
                              gpu_active->device_id,
                              driver_version,
                              pixel_shader_version,
                              vertex_shader_version,
                              gl_version,
                              can_lose_context);
    gpu_info->SetProgress(GPUInfo::kComplete);
  }
  (interface->pci_cleanup)(access);
  FinalizeLibPci(interface);
  FinalizeGLContext(context);
  return (gpu_active != NULL);
}

}  // namespace gpu_info_collector
<|endoftext|>


<commit_before>/*****************************************************************************
 * Project: RooFit                                                           *
 * Package: RooFitCore                                                       *
 * @(#)root/roofitcore:$Id$
 * Authors:                                                                  *
 *   WV, Wouter Verkerke, UC Santa Barbara, verkerke@slac.stanford.edu       *
 *   DK, David Kirkby,    UC Irvine,         dkirkby@uci.edu                 *
 *                                                                           *
 * Copyright (c) 2000-2005, Regents of the University of California          *
 *                          and Stanford University. All rights reserved.    *
 *                                                                           *
 * Redistribution and use in source and binary forms,                        *
 * with or without modification, are permitted according to the terms        *
 * listed in LICENSE (http://roofit.sourceforge.net/license.txt)             *
 *****************************************************************************/

// -- CLASS DESCRIPTION [PDF] --
// RooAbsOptTestStatistic is the abstract base class for goodness-of-fit
// variables that evaluate the PDF at each point of a given dataset.
// This class provides generic optimizations, such as caching and precalculation
// of constant terms that can be made for all such quantities
//
// Implementations should define evaluatePartition(), which calculates the
// value of a (sub)range of the dataset and optionally combinedValue(),
// which combines the values calculated for each partition. If combinedValue()
// is overloaded, the default implementation will add the partition results
// to obtain the combined result
//
// Support for calculation in partitions is needed to allow parallel calculation
// of goodness-of-fit values.

#include "RooFit.h"

#include "Riostream.h"

#include "RooAbsOptTestStatistic.h"
#include "RooMsgService.h"
#include "RooAbsPdf.h"
#include "RooAbsData.h"
#include "RooArgSet.h"
#include "RooRealVar.h"
#include "RooErrorHandler.h"
#include "RooGlobalFunc.h"

ClassImp(RooAbsOptTestStatistic)
;

RooAbsOptTestStatistic:: RooAbsOptTestStatistic()
{
  _normSet = 0 ;
  _pdfCloneSet = 0 ;
  _dataClone = 0 ;
  _pdfClone = 0 ;
  _projDeps = 0 ;
}

RooAbsOptTestStatistic::RooAbsOptTestStatistic(const char *name, const char *title, RooAbsPdf& pdf, RooAbsData& data,
					 const RooArgSet& projDeps, const char* rangeName, const char* addCoefRangeName,
					       Int_t nCPU, Bool_t verbose, Bool_t splitCutRange) :
  RooAbsTestStatistic(name,title,pdf,data,projDeps,rangeName, addCoefRangeName, nCPU, verbose, splitCutRange),
  _projDeps(0)
{

  // Don't do a thing in master mode
  if (operMode()!=Slave) {
    _normSet = 0 ;
    return ;
  }

  RooArgSet obs(*data.get()) ;
  obs.remove(projDeps,kTRUE,kTRUE) ;

  // Check that the PDF is valid for use with this dataset
  // Check if there are any unprotected multiple occurrences of dependents
  if (pdf.recursiveCheckObservables(&obs)) {
    coutE(InputArguments) << "RooAbsOptTestStatistic: ERROR in PDF dependents, abort" << endl ;
    RooErrorHandler::softAbort() ;
    return ;
  }

  // Check if the fit ranges of the dependents in the data and in the PDF are consistent
  RooArgSet* pdfDepSet = pdf.getObservables(&data) ;
  const RooArgSet* dataDepSet = data.get() ;
  TIterator* iter = pdfDepSet->createIterator() ;
  RooAbsArg* arg ;
  while((arg=(RooAbsArg*)iter->Next())) {
    RooRealVar* pdfReal = dynamic_cast<RooRealVar*>(arg) ;
    if (!pdfReal) continue ;

    RooRealVar* datReal = dynamic_cast<RooRealVar*>(dataDepSet->find(pdfReal->GetName())) ;
    if (!datReal) continue ;

    if (pdfReal->getMin()<(datReal->getMin()-1e-6)) {
      coutE(InputArguments) << "RooAbsOptTestStatistic: ERROR minimum of PDF variable " << arg->GetName()
			    << "(" << pdfReal->getMin() << ") is smaller than that of "
			    << arg->GetName() << " in the dataset (" << datReal->getMin() << ")" << endl ;
      RooErrorHandler::softAbort() ;
      return ;
    }

    if (pdfReal->getMax()>(datReal->getMax()+1e-6)) {
      coutE(InputArguments) << "RooAbsOptTestStatistic: ERROR maximum of PDF variable " << arg->GetName()
			    << " is smaller than that of " << arg->GetName() << " in the dataset" << endl ;
      RooErrorHandler::softAbort() ;
      return ;
    }

  }

  // Copy data and strip entries lost by adjusted fit range, _dataClone ranges will be copied from pdfDepSet ranges
  if (rangeName) {
    _dataClone = ((RooAbsData&)data).reduce(RooFit::SelectVars(*pdfDepSet),RooFit::CutRange(rangeName)) ;
  } else {
    _dataClone = ((RooAbsData&)data).reduce(RooFit::SelectVars(*pdfDepSet)) ;
  }

  if (rangeName) {

    cxcoutI(Fitting) << "RooAbsOptTestStatistic::ctor(" << GetName() << ") constructing likelihood for sub-range named " << rangeName << endl ;

    // Adjust PDF normalization ranges to requested fitRange, store original ranges for RooAddPdf coefficient interpretation
    TIterator* iter2 = _dataClone->get()->createIterator() ;
    while((arg=(RooAbsArg*)iter2->Next())) {
      RooRealVar* pdfReal = dynamic_cast<RooRealVar*>(arg) ;
      if (pdfReal) {
	if (!addCoefRangeName) {
	  pdfReal->setRange(Form("NormalizationRangeFor%s",rangeName),pdfReal->getMin(),pdfReal->getMax()) ;
	}
	pdfReal->setRange(pdfReal->getMin(rangeName),pdfReal->getMax(rangeName)) ;
      }
    }

    // Mark fitted range in original PDF dependents for future use
    if (!_splitRange) {
      iter->Reset() ;
      while((arg=(RooAbsArg*)iter->Next())) {
	RooRealVar* pdfReal = dynamic_cast<RooRealVar*>(arg) ;
	if (pdfReal) {
	  pdfReal->setRange("fit",pdfReal->getMin(rangeName),pdfReal->getMax(rangeName)) ;
	}
      }
    }
    delete iter2 ;
  }
  delete iter ;

  delete pdfDepSet ;

  setEventCount(_dataClone->numEntries()) ;

  // Clone all PDF compents by copying all branch nodes
  RooArgSet tmp("PdfBranchNodeList") ;
  pdf.branchNodeServerList(&tmp) ;
  _pdfCloneSet = (RooArgSet*) tmp.snapshot(kFALSE) ;

  // Find the top level PDF in the snapshot list
  _pdfClone = (RooAbsPdf*) _pdfCloneSet->find(pdf.GetName()) ;

  // Fix RooAddPdf coefficients to original normalization range
  if (rangeName) {
    // WVE Remove projected dependents from normalization
    _pdfClone->fixAddCoefNormalization(*_dataClone->get()) ;

    if (addCoefRangeName) {
      cxcoutI(Fitting) << "RooAbsOptTestStatistic::ctor(" << GetName() << ") fixing interpretation of coefficients of any RooAddPdf component to range " << addCoefRangeName << endl ;
      _pdfClone->fixAddCoefRange(addCoefRangeName,kFALSE) ;
    } else {
      cxcoutI(Fitting) << "RooAbsOptTestStatistic::ctor(" << GetName() << ") fixing interpretation of coefficients of any RooAddPdf to full domain of observables " << endl ;
      _pdfClone->fixAddCoefRange(Form("NormalizationRangeFor%s",rangeName),kFALSE) ;
    }
  }

  // Attach PDF to data set
  _pdfClone->attachDataSet(*_dataClone) ;

  // Store normalization set
  _normSet = (RooArgSet*) data.get()->snapshot(kFALSE) ;

  // Remove projected dependents from normalization set
  if (projDeps.getSize()>0) {

    _projDeps = (RooArgSet*) projDeps.snapshot(kFALSE) ;

    RooArgSet* tobedel = (RooArgSet*) _normSet->selectCommon(*_projDeps) ;
    _normSet->remove(*_projDeps,kTRUE,kTRUE) ;

    // Delete owned projected dependent copy in _normSet
    TIterator* ii = tobedel->createIterator() ;
    RooAbsArg* aa ;
    while((aa=(RooAbsArg*)ii->Next())) {
      delete aa ;
    }
    delete ii ;
    delete tobedel ;

    // Mark all projected dependents as such
    RooArgSet *projDataDeps = (RooArgSet*) _dataClone->get()->selectCommon(*_projDeps) ;
    projDataDeps->setAttribAll("projectedDependent") ;
    delete projDataDeps ;
  }

  // Add parameters as servers
  RooArgSet* params = _pdfClone->getParameters(_dataClone) ;
  _paramSet.add(*params) ;
  delete params ;

  // Mark all projected dependents as such
  if (_projDeps) {
    RooArgSet *projDataDeps = (RooArgSet*) _dataClone->get()->selectCommon(*_projDeps) ;
    projDataDeps->setAttribAll("projectedDependent") ;
    delete projDataDeps ;
  }

  coutI(Optimization) << "RooAbsOptTestStatistic::ctor(" << GetName() << ") optimizing internal clone of p.d.f for likelihood evaluation."
			<< "Lazy evaluation and associated change tracking will disabled for all nodes that depend on observables" << endl ;

  optimizeCaching() ;

}

RooAbsOptTestStatistic::RooAbsOptTestStatistic(const RooAbsOptTestStatistic& other, const char* name) :
  RooAbsTestStatistic(other,name)
{
  // Don't do a thing in master mode
  if (operMode()!=Slave) {
    _projDeps = 0 ;
    _normSet = other._normSet ? ((RooArgSet*) other._normSet->snapshot()) : 0 ;
    return ;
  }

  _pdfCloneSet = (RooArgSet*) other._pdfCloneSet->snapshot(kFALSE) ;
  _pdfClone = (RooAbsPdf*) _pdfCloneSet->find(other._pdfClone->GetName()) ;

  // Copy the operMode attribute of all branch nodes
  TIterator* iter = _pdfCloneSet->createIterator() ;
  RooAbsArg* branch ;
  while((branch=(RooAbsArg*)iter->Next())) {
    branch->setOperMode(other._pdfCloneSet->find(branch->GetName())->operMode()) ;
  }
  delete iter ;

  // WVE Must use clone with cache redirection here
  _dataClone = (RooAbsData*) other._dataClone->cacheClone(_pdfCloneSet) ;

  _pdfClone->attachDataSet(*_dataClone) ;
  _normSet = (RooArgSet*) other._normSet->snapshot() ;

  if (other._projDeps) {
    _projDeps = (RooArgSet*) other._projDeps->snapshot() ;
  } else {
    _projDeps = 0 ;
  }
}

RooAbsOptTestStatistic::~RooAbsOptTestStatistic()
{
  if (operMode()==Slave) {
    delete _pdfCloneSet ;
    delete _dataClone ;
    delete _projDeps ;
  }
  delete _normSet ;
}

Double_t RooAbsOptTestStatistic::combinedValue(RooAbsReal** array, Int_t n) const
{
  // Default implementation returns sum of components
  Double_t sum(0) ;
  Int_t i ;
  for (i=0 ; i<n ; i++) {
    Double_t tmp = array[i]->getVal() ;
    if (tmp==0) return 0 ;
    sum += tmp ;
  }
  return sum ;
}

Bool_t RooAbsOptTestStatistic::redirectServersHook(const RooAbsCollection& newServerList, Bool_t mustReplaceAll, Bool_t nameChange, Bool_t isRecursive)
{
  RooAbsTestStatistic::redirectServersHook(newServerList,mustReplaceAll,nameChange,isRecursive) ;
  if (operMode()!=Slave) return kFALSE ;
  Bool_t ret = _pdfClone->recursiveRedirectServers(newServerList,kFALSE,nameChange) ;
  return ret ;
}

void RooAbsOptTestStatistic::printCompactTreeHook(ostream& os, const char* indent)
{
  RooAbsTestStatistic::printCompactTreeHook(os,indent) ;
  if (operMode()!=Slave) return ;
  TString indent2(indent) ;
  indent2 += ">>" ;
  _pdfClone->printCompactTree(os,indent2) ;
}

void RooAbsOptTestStatistic::constOptimizeTestStatistic(ConstOpCode opcode)
{
  // Driver function to propagate const optimization
  RooAbsTestStatistic::constOptimizeTestStatistic(opcode);
  if (operMode()!=Slave) return ;

  switch(opcode) {
  case Activate:
    cxcoutI(Optimization) << "RooAbsOptTestStatistic::constOptimize(" << GetName() << ") optimizing evaluation of test statistic by finding all nodes in p.d.f that depend exclusively"
			    << " on observables and constant parameters and precalculating their values" << endl ;
    optimizeConstantTerms(kTRUE) ;
    break ;
  case DeActivate:
    cxcoutI(Optimization) << "RooAbsOptTestStatistic::constOptimize(" << GetName() << ") deactivating optimization of constant terms in test statistic" << endl ;
    optimizeConstantTerms(kFALSE) ;
    break ;
  case ConfigChange:
    cxcoutI(Optimization) << "RooAbsOptTestStatistic::constOptimize(" << GetName() << ") one ore more parameter were changed from constant to floating or vice versa, "
			    << "re-evaluating constant term optimization" << endl ;
    optimizeConstantTerms(kFALSE) ;
    optimizeConstantTerms(kTRUE) ;
    break ;
  case ValueChange:
    cxcoutI(Optimization) << "RooAbsOptTestStatistic::constOptimize(" << GetName() << ") the value of one ore more constant parameter were changed re-evaluating constant term optimization" << endl ;
    optimizeConstantTerms(kFALSE) ;
    optimizeConstantTerms(kTRUE) ;
    break ;
  }
}

void RooAbsOptTestStatistic::optimizeCaching()
{
  // This method changes the value caching logic for all nodes that depends on any of the observables
  // as defined by the given dataset. When evaluating a test statistic constructed from the RooAbsReal
  // with a dataset the observables are guaranteed to change with every call, thus there is no point
  // in tracking these changes which result in a net overhead. Thus for observable-dependent nodes,
  // the evaluation mechanism is changed from being dependent on a 'valueDirty' flag to guaranteed evaluation.
  // On the dataset side, the observables objects are modified to no longer send valueDirty messages
  // to their client

  // Trigger create of all object caches now in nodes that have deferred object creation
  // so that cache contents can be processed immediately
  _pdfClone->getVal(_normSet) ;

  // Set value caching mode for all nodes that depend on any of the observables to ADirty
  _pdfClone->optimizeCacheMode(*_dataClone->get()) ;

  // Disable propagation of dirty state flags for observables
  _dataClone->setDirtyProp(kFALSE) ;

  // Disable reading of observables that are not used
  _dataClone->optimizeReadingWithCaching(*_pdfClone, RooArgSet()) ;
}

void RooAbsOptTestStatistic::optimizeConstantTerms(Bool_t activate)
{
  if(activate) {
    // Trigger create of all object caches now in nodes that have deferred object creation
    // so that cache contents can be processed immediately
    _pdfClone->getVal(_normSet) ;

    // Find all nodes that depend exclusively on constant parameters
    RooArgSet cacheableNodes ;
    _pdfClone->findConstantNodes(*_dataClone->get(),cacheableNodes) ;

    // Cache constant nodes with dataset
    _dataClone->cacheArgs(cacheableNodes,_normSet) ;

    // Put all cached nodes in AClean value caching mode so that their evaluate() is never called
    TIterator* cIter = cacheableNodes.createIterator() ;
    RooAbsArg *cacheArg ;
    while((cacheArg=(RooAbsArg*)cIter->Next())){
      cacheArg->setOperMode(RooAbsArg::AClean) ;
    }
    delete cIter ;

    // Disable reading of observables that are no longer used
    _dataClone->optimizeReadingWithCaching(*_pdfClone, cacheableNodes) ;

  } else {

    // Delete the cache
    _dataClone->resetCache() ;

    // Reactivate all tree branches
    _dataClone->setArgStatus(*_dataClone->get(),kTRUE) ;

    // Reset all nodes to ADirty
    optimizeCaching() ;

  }
}

<commit_msg>From Wouter: Protect against null strings<commit_after>/*****************************************************************************
 * Project: RooFit                                                           *
 * Package: RooFitCore                                                       *
 * @(#)root/roofitcore:$Id$
 * Authors:                                                                  *
 *   WV, Wouter Verkerke, UC Santa Barbara, verkerke@slac.stanford.edu       *
 *   DK, David Kirkby,    UC Irvine,         dkirkby@uci.edu                 *
 *                                                                           *
 * Copyright (c) 2000-2005, Regents of the University of California          *
 *                          and Stanford University. All rights reserved.    *
 *                                                                           *
 * Redistribution and use in source and binary forms,                        *
 * with or without modification, are permitted according to the terms        *
 * listed in LICENSE (http://roofit.sourceforge.net/license.txt)             *
 *****************************************************************************/

// -- CLASS DESCRIPTION [PDF] --
// RooAbsOptTestStatistic is the abstract base class for goodness-of-fit
// variables that evaluate the PDF at each point of a given dataset.
// This class provides generic optimizations, such as caching and precalculation
// of constant terms that can be made for all such quantities
//
// Implementations should define evaluatePartition(), which calculates the
// value of a (sub)range of the dataset and optionally combinedValue(),
// which combines the values calculated for each partition. If combinedValue()
// is overloaded, the default implementation will add the partition results
// to obtain the combined result
//
// Support for calculation in partitions is needed to allow parallel calculation
// of goodness-of-fit values.

#include "RooFit.h"

#include "Riostream.h"
#include <string.h>

#include "RooAbsOptTestStatistic.h"
#include "RooMsgService.h"
#include "RooAbsPdf.h"
#include "RooAbsData.h"
#include "RooArgSet.h"
#include "RooRealVar.h"
#include "RooErrorHandler.h"
#include "RooGlobalFunc.h"

ClassImp(RooAbsOptTestStatistic)
;

RooAbsOptTestStatistic:: RooAbsOptTestStatistic()
{
  _normSet = 0 ;
  _pdfCloneSet = 0 ;
  _dataClone = 0 ;
  _pdfClone = 0 ;
  _projDeps = 0 ;
}

RooAbsOptTestStatistic::RooAbsOptTestStatistic(const char *name, const char *title, RooAbsPdf& pdf, RooAbsData& data,
					 const RooArgSet& projDeps, const char* rangeName, const char* addCoefRangeName,
					       Int_t nCPU, Bool_t verbose, Bool_t splitCutRange) :
  RooAbsTestStatistic(name,title,pdf,data,projDeps,rangeName, addCoefRangeName, nCPU, verbose, splitCutRange),
  _projDeps(0)
{

  // Don't do a thing in master mode
  if (operMode()!=Slave) {
    _normSet = 0 ;
    return ;
  }

  RooArgSet obs(*data.get()) ;
  obs.remove(projDeps,kTRUE,kTRUE) ;

  // Check that the PDF is valid for use with this dataset
  // Check if there are any unprotected multiple occurrences of dependents
  if (pdf.recursiveCheckObservables(&obs)) {
    coutE(InputArguments) << "RooAbsOptTestStatistic: ERROR in PDF dependents, abort" << endl ;
    RooErrorHandler::softAbort() ;
    return ;
  }

  // Check if the fit ranges of the dependents in the data and in the PDF are consistent
  RooArgSet* pdfDepSet = pdf.getObservables(&data) ;
  const RooArgSet* dataDepSet = data.get() ;
  TIterator* iter = pdfDepSet->createIterator() ;
  RooAbsArg* arg ;
  while((arg=(RooAbsArg*)iter->Next())) {
    RooRealVar* pdfReal = dynamic_cast<RooRealVar*>(arg) ;
    if (!pdfReal) continue ;

    RooRealVar* datReal = dynamic_cast<RooRealVar*>(dataDepSet->find(pdfReal->GetName())) ;
    if (!datReal) continue ;

    if (pdfReal->getMin()<(datReal->getMin()-1e-6)) {
      coutE(InputArguments) << "RooAbsOptTestStatistic: ERROR minimum of PDF variable " << arg->GetName()
			    << "(" << pdfReal->getMin() << ") is smaller than that of "
			    << arg->GetName() << " in the dataset (" << datReal->getMin() << ")" << endl ;
      RooErrorHandler::softAbort() ;
      return ;
    }

    if (pdfReal->getMax()>(datReal->getMax()+1e-6)) {
      coutE(InputArguments) << "RooAbsOptTestStatistic: ERROR maximum of PDF variable " << arg->GetName()
			    << " is smaller than that of " << arg->GetName() << " in the dataset" << endl ;
      RooErrorHandler::softAbort() ;
      return ;
    }

  }

  // Copy data and strip entries lost by adjusted fit range, _dataClone ranges will be copied from pdfDepSet ranges
  if (rangeName && strlen(rangeName)) {
    _dataClone = ((RooAbsData&)data).reduce(RooFit::SelectVars(*pdfDepSet),RooFit::CutRange(rangeName)) ;
  } else {
    _dataClone = ((RooAbsData&)data).reduce(RooFit::SelectVars(*pdfDepSet)) ;
  }

  if (rangeName && strlen(rangeName)) {

    cxcoutI(Fitting) << "RooAbsOptTestStatistic::ctor(" << GetName() << ") constructing likelihood for sub-range named " << rangeName << endl ;

    // Adjust PDF normalization ranges to requested fitRange, store original ranges for RooAddPdf coefficient interpretation
    TIterator* iter2 = _dataClone->get()->createIterator() ;
    while((arg=(RooAbsArg*)iter2->Next())) {
      RooRealVar* pdfReal = dynamic_cast<RooRealVar*>(arg) ;
      if (pdfReal) {
        if (!(addCoefRangeName && strlen(addCoefRangeName))) {
	  pdfReal->setRange(Form("NormalizationRangeFor%s",rangeName),pdfReal->getMin(),pdfReal->getMax()) ;
	}
	pdfReal->setRange(pdfReal->getMin(rangeName),pdfReal->getMax(rangeName)) ;
      }
    }

    // Mark fitted range in original PDF dependents for future use
    if (!_splitRange) {
      iter->Reset() ;
      while((arg=(RooAbsArg*)iter->Next())) {
	RooRealVar* pdfReal = dynamic_cast<RooRealVar*>(arg) ;
	if (pdfReal) {
	  pdfReal->setRange("fit",pdfReal->getMin(rangeName),pdfReal->getMax(rangeName)) ;
	}
      }
    }
    delete iter2 ;
  }
  delete iter ;

  delete pdfDepSet ;

  setEventCount(_dataClone->numEntries()) ;

  // Clone all PDF compents by copying all branch nodes
  RooArgSet tmp("PdfBranchNodeList") ;
  pdf.branchNodeServerList(&tmp) ;
  _pdfCloneSet = (RooArgSet*) tmp.snapshot(kFALSE) ;

  // Find the top level PDF in the snapshot list
  _pdfClone = (RooAbsPdf*) _pdfCloneSet->find(pdf.GetName()) ;

  // Fix RooAddPdf coefficients to original normalization range
  if (rangeName) {
    // WVE Remove projected dependents from normalization
    _pdfClone->fixAddCoefNormalization(*_dataClone->get()) ;

    if (addCoefRangeName) {
      cxcoutI(Fitting) << "RooAbsOptTestStatistic::ctor(" << GetName() << ") fixing interpretation of coefficients of any RooAddPdf component to range " << addCoefRangeName << endl ;
      _pdfClone->fixAddCoefRange(addCoefRangeName,kFALSE) ;
    } else {
      cxcoutI(Fitting) << "RooAbsOptTestStatistic::ctor(" << GetName() << ") fixing interpretation of coefficients of any RooAddPdf to full domain of observables " << endl ;
      _pdfClone->fixAddCoefRange(Form("NormalizationRangeFor%s",rangeName),kFALSE) ;
    }
  }

  // Attach PDF to data set
  _pdfClone->attachDataSet(*_dataClone) ;

  // Store normalization set
  _normSet = (RooArgSet*) data.get()->snapshot(kFALSE) ;

  // Remove projected dependents from normalization set
  if (projDeps.getSize()>0) {

    _projDeps = (RooArgSet*) projDeps.snapshot(kFALSE) ;

    RooArgSet* tobedel = (RooArgSet*) _normSet->selectCommon(*_projDeps) ;
    _normSet->remove(*_projDeps,kTRUE,kTRUE) ;

    // Delete owned projected dependent copy in _normSet
    TIterator* ii = tobedel->createIterator() ;
    RooAbsArg* aa ;
    while((aa=(RooAbsArg*)ii->Next())) {
      delete aa ;
    }
    delete ii ;
    delete tobedel ;

    // Mark all projected dependents as such
    RooArgSet *projDataDeps = (RooArgSet*) _dataClone->get()->selectCommon(*_projDeps) ;
    projDataDeps->setAttribAll("projectedDependent") ;
    delete projDataDeps ;
  }

  // Add parameters as servers
  RooArgSet* params = _pdfClone->getParameters(_dataClone) ;
  _paramSet.add(*params) ;
  delete params ;

  // Mark all projected dependents as such
  if (_projDeps) {
    RooArgSet *projDataDeps = (RooArgSet*) _dataClone->get()->selectCommon(*_projDeps) ;
    projDataDeps->setAttribAll("projectedDependent") ;
    delete projDataDeps ;
  }

  coutI(Optimization) << "RooAbsOptTestStatistic::ctor(" << GetName() << ") optimizing internal clone of p.d.f for likelihood evaluation."
			<< "Lazy evaluation and associated change tracking will disabled for all nodes that depend on observables" << endl ;

  optimizeCaching() ;

}

RooAbsOptTestStatistic::RooAbsOptTestStatistic(const RooAbsOptTestStatistic& other, const char* name) :
  RooAbsTestStatistic(other,name)
{
  // Don't do a thing in master mode
  if (operMode()!=Slave) {
    _projDeps = 0 ;
    _normSet = other._normSet ? ((RooArgSet*) other._normSet->snapshot()) : 0 ;
    return ;
  }

  _pdfCloneSet = (RooArgSet*) other._pdfCloneSet->snapshot(kFALSE) ;
  _pdfClone = (RooAbsPdf*) _pdfCloneSet->find(other._pdfClone->GetName()) ;

  // Copy the operMode attribute of all branch nodes
  TIterator* iter = _pdfCloneSet->createIterator() ;
  RooAbsArg* branch ;
  while((branch=(RooAbsArg*)iter->Next())) {
    branch->setOperMode(other._pdfCloneSet->find(branch->GetName())->operMode()) ;
  }
  delete iter ;

  // WVE Must use clone with cache redirection here
  _dataClone = (RooAbsData*) other._dataClone->cacheClone(_pdfCloneSet) ;

  _pdfClone->attachDataSet(*_dataClone) ;
  _normSet = (RooArgSet*) other._normSet->snapshot() ;

  if (other._projDeps) {
    _projDeps = (RooArgSet*) other._projDeps->snapshot() ;
  } else {
    _projDeps = 0 ;
  }
}

RooAbsOptTestStatistic::~RooAbsOptTestStatistic()
{
  if (operMode()==Slave) {
    delete _pdfCloneSet ;
    delete _dataClone ;
    delete _projDeps ;
  }
  delete _normSet ;
}

Double_t RooAbsOptTestStatistic::combinedValue(RooAbsReal** array, Int_t n) const
{
  // Default implementation returns sum of components
  Double_t sum(0) ;
  Int_t i ;
  for (i=0 ; i<n ; i++) {
    Double_t tmp = array[i]->getVal() ;
    if (tmp==0) return 0 ;
    sum += tmp ;
  }
  return sum ;
}

Bool_t RooAbsOptTestStatistic::redirectServersHook(const RooAbsCollection& newServerList, Bool_t mustReplaceAll, Bool_t nameChange, Bool_t isRecursive)
{
  RooAbsTestStatistic::redirectServersHook(newServerList,mustReplaceAll,nameChange,isRecursive) ;
  if (operMode()!=Slave) return kFALSE ;
  Bool_t ret = _pdfClone->recursiveRedirectServers(newServerList,kFALSE,nameChange) ;
  return ret ;
}

void RooAbsOptTestStatistic::printCompactTreeHook(ostream& os, const char* indent)
{
  RooAbsTestStatistic::printCompactTreeHook(os,indent) ;
  if (operMode()!=Slave) return ;
  TString indent2(indent) ;
  indent2 += ">>" ;
  _pdfClone->printCompactTree(os,indent2) ;
}

void RooAbsOptTestStatistic::constOptimizeTestStatistic(ConstOpCode opcode)
{
  // Driver function to propagate const optimization
  RooAbsTestStatistic::constOptimizeTestStatistic(opcode);
  if (operMode()!=Slave) return ;

  switch(opcode) {
  case Activate:
    cxcoutI(Optimization) << "RooAbsOptTestStatistic::constOptimize(" << GetName() << ") optimizing evaluation of test statistic by finding all nodes in p.d.f that depend exclusively"
			    << " on observables and constant parameters and precalculating their values" << endl ;
    optimizeConstantTerms(kTRUE) ;
    break ;
  case DeActivate:
    cxcoutI(Optimization) << "RooAbsOptTestStatistic::constOptimize(" << GetName() << ") deactivating optimization of constant terms in test statistic" << endl ;
    optimizeConstantTerms(kFALSE) ;
    break ;
  case ConfigChange:
    cxcoutI(Optimization) << "RooAbsOptTestStatistic::constOptimize(" << GetName() << ") one ore more parameter were changed from constant to floating or vice versa, "
			    << "re-evaluating constant term optimization" << endl ;
    optimizeConstantTerms(kFALSE) ;
    optimizeConstantTerms(kTRUE) ;
    break ;
  case ValueChange:
    cxcoutI(Optimization) << "RooAbsOptTestStatistic::constOptimize(" << GetName() << ") the value of one ore more constant parameter were changed re-evaluating constant term optimization" << endl ;
    optimizeConstantTerms(kFALSE) ;
    optimizeConstantTerms(kTRUE) ;
    break ;
  }
}

void RooAbsOptTestStatistic::optimizeCaching()
{
  // This method changes the value caching logic for all nodes that depends on any of the observables
  // as defined by the given dataset. When evaluating a test statistic constructed from the RooAbsReal
  // with a dataset the observables are guaranteed to change with every call, thus there is no point
  // in tracking these changes which result in a net overhead. Thus for observable-dependent nodes,
  // the evaluation mechanism is changed from being dependent on a 'valueDirty' flag to guaranteed evaluation.
  // On the dataset side, the observables objects are modified to no longer send valueDirty messages
  // to their client

  // Trigger create of all object caches now in nodes that have deferred object creation
  // so that cache contents can be processed immediately
  _pdfClone->getVal(_normSet) ;

  // Set value caching mode for all nodes that depend on any of the observables to ADirty
  _pdfClone->optimizeCacheMode(*_dataClone->get()) ;

  // Disable propagation of dirty state flags for observables
  _dataClone->setDirtyProp(kFALSE) ;

  // Disable reading of observables that are not used
  _dataClone->optimizeReadingWithCaching(*_pdfClone, RooArgSet()) ;
}

void RooAbsOptTestStatistic::optimizeConstantTerms(Bool_t activate)
{
  if(activate) {
    // Trigger create of all object caches now in nodes that have deferred object creation
    // so that cache contents can be processed immediately
    _pdfClone->getVal(_normSet) ;

    // Find all nodes that depend exclusively on constant parameters
    RooArgSet cacheableNodes ;
    _pdfClone->findConstantNodes(*_dataClone->get(),cacheableNodes) ;

    // Cache constant nodes with dataset
    _dataClone->cacheArgs(cacheableNodes,_normSet) ;

    // Put all cached nodes in AClean value caching mode so that their evaluate() is never called
    TIterator* cIter = cacheableNodes.createIterator() ;
    RooAbsArg *cacheArg ;
    while((cacheArg=(RooAbsArg*)cIter->Next())){
      cacheArg->setOperMode(RooAbsArg::AClean) ;
    }
    delete cIter ;

    // Disable reading of observables that are no longer used
    _dataClone->optimizeReadingWithCaching(*_pdfClone, cacheableNodes) ;

  } else {

    // Delete the cache
    _dataClone->resetCache() ;

    // Reactivate all tree branches
    _dataClone->setArgStatus(*_dataClone->get(),kTRUE) ;

    // Reset all nodes to ADirty
    optimizeCaching() ;

  }
}

<|endoftext|>


<commit_before>/**********************************************************************
* Copyright (c) 2013-2014  Red Hat, Inc.
*
* Developed by Daynix Computing LTD.
*
* Authors:
*     Dmitry Fleytman <dmitry@daynix.com>
*     Pavel Gurvich <pavel@daynix.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
**********************************************************************/

#include "stdafx.h"

#if TARGET_OS_WIN_XP
NTSTATUS WdfUsbTargetDeviceCreateIsochUrb(WDFUSBDEVICE UsbDevice, PWDF_OBJECT_ATTRIBUTES Attributes, ULONG NumberOfIsochPackets,
    WDFMEMORY* UrbMemory, PURB *Urb)
{
    UNREFERENCED_PARAMETER(UsbDevice);

    size_t size = GET_ISO_URB_SIZE(NumberOfIsochPackets);
    auto status = WdfMemoryCreate(Attributes, NonPagedPool, 'SBSU', size, UrbMemory, (PVOID*)Urb);
    if (NT_SUCCESS(status))
    {
        RtlZeroMemory(Urb, size);
    }

    return status;
}
#endif
<commit_msg>UsbDk: BZ#1139434 Windows XP crashes after some time of a realtime USB traffic (webcam)<commit_after>/**********************************************************************
* Copyright (c) 2013-2014  Red Hat, Inc.
*
* Developed by Daynix Computing LTD.
*
* Authors:
*     Dmitry Fleytman <dmitry@daynix.com>
*     Pavel Gurvich <pavel@daynix.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
**********************************************************************/

#include "stdafx.h"

#if TARGET_OS_WIN_XP
NTSTATUS WdfUsbTargetDeviceCreateIsochUrb(WDFUSBDEVICE UsbDevice, PWDF_OBJECT_ATTRIBUTES Attributes, ULONG NumberOfIsochPackets,
    WDFMEMORY* UrbMemory, PURB *Urb)
{
    UNREFERENCED_PARAMETER(UsbDevice);

    size_t size = GET_ISO_URB_SIZE(NumberOfIsochPackets);
    auto status = WdfMemoryCreate(Attributes, NonPagedPool, 'SBSU', size, UrbMemory, (PVOID*)Urb);
    if (NT_SUCCESS(status))
    {
        RtlZeroMemory(*Urb, size);
    }

    return status;
}
#endif
<|endoftext|>


<commit_before>#pragma once

#include "../util.hpp"

template<int M, bool IsPrime = false>
class Modulo {
  using ll = long long;
  int n;
  static enable_if_t<IsPrime, ll> inv(ll a, ll p) {
    return (a == 1 ? 1 : (1 - p * inv(p%a, a)) / a + p);
  }
public:
  Modulo () : n(0) {;}
  Modulo (int m) : n(m) {
    if (n >= M) n %= M;
    else if (n < 0) n = (n % M + M) % M;
  }
  Modulo (ll m) {
    if (m >= M) m %= M;
    else if (m < 0) m = (m % M + M) % M;
    n = m;
  }
  explicit operator int() const { return n; }
  explicit operator ll() const { return n; }
  bool operator==(const Modulo &a) const { return n == a.n; }
  Modulo &operator+=(const Modulo &a) { n += a.n; if (n >= M) n -= M; return *this; }
  Modulo &operator-=(const Modulo &a) { n -= a.n; if (n < 0) n += M; return *this; }
  Modulo &operator*=(const Modulo &a) { n = (ll(n) * a.n) % M; return *this; }
  Modulo operator+(const Modulo &a) const { Modulo res = *this; return res += a; }
  Modulo operator-(const Modulo &a) const { Modulo res = *this; return res -= a; }
  Modulo operator*(const Modulo &a) const { Modulo res = *this; return res *= a; }
  Modulo operator^(int m) const {
    if (m == 0) return Modulo(1);
    const Modulo a = *this;
    Modulo res = (a * a) ^ (m / 2);
    return m % 2 ? res * a : res;
  }
  enable_if_t<IsPrime, Modulo> operator/(const Modulo &a) const {
    return *this * inv(ll(a), M);
  }
  enable_if_t<IsPrime, Modulo> operator/=(const Modulo &a) {
    return *this *= inv(ll(a), M);
  }
};

template<int M, bool IsPrime = false>
bool is_zero(Modulo<M, IsPrime> x) { return int(x) == 0; }
template<int M, bool IsPrime = false>
int abs(Modulo<M, IsPrime> x) { return int(x); }

const int mod = 1000000007;

template<int M = mod> Modulo<M, true> fact(int n, bool sw = true) {
  static vector<Modulo<M, true>> v1 = {1}, v2 = {1};
  if (n >= (int)v1.size()) {
    const int from = v1.size(), to = n + 1024;
    v1.reserve(to);
    v2.reserve(to);
    for (int i = from; i < to; ++i) {
      v1.push_back(v1.back() * Modulo<M, true>(i));
      v2.push_back(v2.back() / Modulo<M, true>(i));
    }
  }
  return sw ? v1[n] : v2[n];
}

template<int M = mod> Modulo<M, true> comb(int a, int b) {
  if (b < 0 || b > a) return 0;
  return fact<M>(a, true) * fact<M>(b, false) * fact<M>(a-b, false);
}

using Mod = Modulo<mod, true>;
<commit_msg>Avoid using enable_if_t<commit_after>#pragma once

#include "../util.hpp"

template<int M, bool IsPrime = false>
class Modulo {
  using ll = long long;
  int n;
  static typename enable_if<IsPrime, ll>::type inv(ll a, ll p) {
    return (a == 1 ? 1 : (1 - p * inv(p%a, a)) / a + p);
  }
public:
  Modulo () : n(0) {;}
  Modulo (int m) : n(m) {
    if (n >= M) n %= M;
    else if (n < 0) n = (n % M + M) % M;
  }
  Modulo (ll m) {
    if (m >= M) m %= M;
    else if (m < 0) m = (m % M + M) % M;
    n = m;
  }
  explicit operator int() const { return n; }
  explicit operator ll() const { return n; }
  bool operator==(const Modulo &a) const { return n == a.n; }
  Modulo &operator+=(const Modulo &a) { n += a.n; if (n >= M) n -= M; return *this; }
  Modulo &operator-=(const Modulo &a) { n -= a.n; if (n < 0) n += M; return *this; }
  Modulo &operator*=(const Modulo &a) { n = (ll(n) * a.n) % M; return *this; }
  Modulo operator+(const Modulo &a) const { Modulo res = *this; return res += a; }
  Modulo operator-(const Modulo &a) const { Modulo res = *this; return res -= a; }
  Modulo operator*(const Modulo &a) const { Modulo res = *this; return res *= a; }
  Modulo operator^(int m) const {
    if (m == 0) return Modulo(1);
    const Modulo a = *this;
    Modulo res = (a * a) ^ (m / 2);
    return m % 2 ? res * a : res;
  }
  typename enable_if<IsPrime, Modulo>::type operator/(const Modulo &a) const {
    return *this * inv(ll(a), M);
  }
  typename enable_if<IsPrime, Modulo>::type operator/=(const Modulo &a) {
    return *this *= inv(ll(a), M);
  }
};

template<int M, bool IsPrime = false>
bool is_zero(Modulo<M, IsPrime> x) { return int(x) == 0; }
template<int M, bool IsPrime = false>
int abs(Modulo<M, IsPrime> x) { return int(x); }

const int mod = 1000000007;

template<int M = mod> Modulo<M, true> fact(int n, bool sw = true) {
  static vector<Modulo<M, true>> v1 = {1}, v2 = {1};
  if (n >= (int)v1.size()) {
    const int from = v1.size(), to = n + 1024;
    v1.reserve(to);
    v2.reserve(to);
    for (int i = from; i < to; ++i) {
      v1.push_back(v1.back() * Modulo<M, true>(i));
      v2.push_back(v2.back() / Modulo<M, true>(i));
    }
  }
  return sw ? v1[n] : v2[n];
}

template<int M = mod> Modulo<M, true> comb(int a, int b) {
  if (b < 0 || b > a) return 0;
  return fact<M>(a, true) * fact<M>(b, false) * fact<M>(a-b, false);
}

using Mod = Modulo<mod, true>;
<|endoftext|>


<commit_before>/** \file    new_journal_alert.cc
 *  \brief   Detects new journal issues for subscribed users.
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2016, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <sstream>
#include <vector>
#include <cstdlib>
#include <cstring>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include "Compiler.h"
#include "DbConnection.h"
#include "EmailSender.h"
#include "ExecUtil.h"
#include "FileUtil.h"
#include "HtmlUtil.h"
#include "MiscUtil.h"
#include "Solr.h"
#include "StringUtil.h"
#include "UrlUtil.h"
#include "util.h"
#include "VuFind.h"

void Usage() {
    std::cerr << "Usage: " << ::progname << " [--verbose] [solr_host_and_port] vufind_config_file_path\n";
    std::cerr << "  Sends out notification emails for journal subscribers.\n";
    std::cerr << "  Should \"solr_host_and_port\" be missing \"localhost:8080\" will be used.\n";
    std::exit(EXIT_FAILURE);
}

struct SerialControlNumberAndLastIssueDate {
    std::string serial_control_number_;
    std::string last_issue_date_;
    bool changed_;
public:
    SerialControlNumberAndLastIssueDate(const std::string &serial_control_number, const std::string &last_issue_date)
        : serial_control_number_(serial_control_number), last_issue_date_(last_issue_date), changed_(false) { }
    inline void setLastIssueDate(const std::string &new_last_issue_date)
        { last_issue_date_ = new_last_issue_date; changed_ = true; }
    inline bool changed() const { return changed_; }
};

struct NewIssueInfo {
    std::string control_number_;
    std::string journal_title_;
    std::string issue_title_;
    std::string last_issue_date_;
public:
    NewIssueInfo(const std::string &control_number, const std::string &journal_title, const std::string &issue_title)
        : control_number_(control_number), journal_title_(journal_title), issue_title_(issue_title) { }
};

/** \return True if new issues were found, false o/w. */
bool ExtractNewIssueInfos(const std::string &json_document, std::vector<NewIssueInfo> * const new_issue_infos,
                          std::string * const max_last_issue_date)
{
    std::stringstream input(json_document, std::ios_base::in);
    boost::property_tree::ptree property_tree;
    boost::property_tree::json_parser::read_json(input, property_tree);

    bool found_at_least_one_new_issue(false);
    for (const auto &document : property_tree.get_child("response.docs.")) {
        const auto &id(document.second.get<std::string>("id"));
        const auto &issue_title(document.second.get<std::string>("title"));
        const std::string journal_title(document.second.get<std::string>("journal_issue/0", "*No Journal Title*"));
        new_issue_infos->emplace_back(id, journal_title, issue_title);

        const auto &recording_date(document.second.get<std::string>("recording_date"));
        if (recording_date > *max_last_issue_date) {
            *max_last_issue_date = recording_date;
            found_at_least_one_new_issue = true;
        }
    }

    return found_at_least_one_new_issue;
}

bool GetNewIssues(const std::string &solr_host_and_port, const std::string &serial_control_number,
                  std::string last_issue_date, std::vector<NewIssueInfo> * const new_issue_infos,
                  std::string * const max_last_issue_date)
{
    if (not StringUtil::EndsWith(last_issue_date, "Z"))
        last_issue_date += "T00:00:00Z"; // Solr does not support the short form of the ISO 8601 date formats.
    const std::string QUERY("superior_ppn:" + serial_control_number + " AND recording_date:{" + last_issue_date
                            + " TO *}");
    std::string json_result;
    if (unlikely(not Solr::Query(QUERY, "id,title,author,recording_date", &json_result, solr_host_and_port,
                                 /* timeout = */ 5, Solr::JSON)))
        Error("Solr query failed or timed-out: \"" + QUERY + "\".");

    return ExtractNewIssueInfos(json_result, new_issue_infos, max_last_issue_date);
}

void SendNotificationEmail(const std::string &firstname, const std::string &lastname, const std::string &email,
                           const std::string &vufind_host, const std::vector<NewIssueInfo> &new_issue_infos)
{
    const std::string EMAIL_TEMPLATE_PATH("/var/lib/tuelib/subscriptions_email.template");
    std::string email_template;
    if (unlikely(not FileUtil::ReadString(EMAIL_TEMPLATE_PATH, &email_template)))
        Error("can't load email template \"" + EMAIL_TEMPLATE_PATH + "\"!");

    // Process the email template:
    std::map<std::string, std::vector<std::string>> names_to_values_map;
    names_to_values_map["firstname"] = std::vector<std::string>{ firstname };
    names_to_values_map["lastname"] = std::vector<std::string>{ lastname };
    std::vector<std::string> urls, journal_titles, issue_titles;
    for (const auto &new_issue_info : new_issue_infos) {
        urls.emplace_back("https://" + vufind_host + "/Record/" + new_issue_info.control_number_);
        journal_titles.emplace_back(new_issue_info.journal_title_);
        issue_titles.emplace_back(HtmlUtil::HtmlEscape(new_issue_info.issue_title_));
    }
    names_to_values_map["url"]           = urls;
    names_to_values_map["journal_title"] = journal_titles;
    names_to_values_map["issue_title"]   = issue_titles;
    std::istringstream input(email_template);
    std::ostringstream email_contents;
    MiscUtil::ExpandTemplate(input, email_contents, names_to_values_map);

    if (unlikely(not EmailSender::SendEmail("notifications@ixtheo.de", email, "Ixtheo Subscriptions",
                                            email_contents.str(), EmailSender::DO_NOT_SET_PRIORITY,
                                            EmailSender::HTML)))
        Error("failed to send a notification email to \"" + email + "\"!");
}

/** \return If "host_and_port" has a colon, the part before the colon else all of "host_and_port". */
std::string GetHostname() {
    std::string hostname;
    if (unlikely(not ExecUtil::ExecSubcommandAndCaptureStdout("/bin/hostname --fqdn", &hostname) or hostname.empty()))
        Error("failed to execute /bin/hostname or got an empty hostname!");
    return hostname;
}

void ProcessSingleUser(const bool verbose, DbConnection * const db_connection, const std::string &user_id,
                       const std::string &solr_host_and_port,
                       std::vector<SerialControlNumberAndLastIssueDate> &control_numbers_and_last_issue_dates)
{
    const std::string SELECT_USER_ATTRIBUTES("SELECT * FROM user WHERE id=" + user_id);
    if (unlikely(not db_connection->query(SELECT_USER_ATTRIBUTES)))
        Error("Select failed: " + SELECT_USER_ATTRIBUTES + " (" + db_connection->getLastErrorMessage() + ")");
    DbResultSet result_set(db_connection->getLastResultSet());
    if (result_set.empty())
        Error("found no user attributes in table \"user\" for ID \"" + user_id + "\"!");
    if (result_set.size() > 1)
        Error("found multiple user attribute sets in table \"user\" for ID \"" + user_id + "\"!");

    const DbRow row(result_set.getNextRow());
    const std::string username(row["username"]);
    if (verbose)
        std::cerr << "Found " << control_numbers_and_last_issue_dates.size() << " subscriptions for \"" << username
                  << ".\n";

    const std::string firstname(row["firstname"]);
    const std::string lastname(row["lastname"]);
    const std::string email(row["email"]);

    // Collect the dates for new issues.
    std::vector<NewIssueInfo> new_issue_infos;
    for (auto &control_number_and_last_issue_date : control_numbers_and_last_issue_dates) {
        std::string max_last_issue_date(control_number_and_last_issue_date.last_issue_date_);
        if (GetNewIssues(solr_host_and_port, control_number_and_last_issue_date.serial_control_number_,
                         control_number_and_last_issue_date.last_issue_date_, &new_issue_infos, &max_last_issue_date))
            control_number_and_last_issue_date.setLastIssueDate(max_last_issue_date);
    }
    if (verbose)
        std::cerr << "Found " << new_issue_infos.size() << " new issues for " << " \"" << username << "\".\n";

    if (not new_issue_infos.empty())
        SendNotificationEmail(firstname, lastname, email, GetHostname(), new_issue_infos);

    // Update the database with the new last issue dates.
    for (const auto &control_number_and_last_issue_date : control_numbers_and_last_issue_dates) {
        if (not control_number_and_last_issue_date.changed())
            continue;

        const std::string REPLACE_STMT("REPLACE INTO ixtheo_journal_subscriptions SET id=" + user_id
                                       + ",last_issue_date='" + control_number_and_last_issue_date.last_issue_date_
                                       + "',journal_control_number='"
                                       + control_number_and_last_issue_date.serial_control_number_ + "'");
        if (unlikely(not db_connection->query(REPLACE_STMT)))
            Error("Replace failed: " + REPLACE_STMT + " (" + db_connection->getLastErrorMessage() + ")");
    }
}

void ProcessSubscriptions(const bool verbose, DbConnection * const db_connection,
                          const std::string &solr_host_and_port)
{
    const std::string SELECT_IDS_STMT("SELECT DISTINCT id FROM ixtheo_journal_subscriptions");
    if (unlikely(not db_connection->query(SELECT_IDS_STMT)))
        Error("Select failed: " + SELECT_IDS_STMT + " (" + db_connection->getLastErrorMessage() + ")");

    unsigned subscription_count(0);
    DbResultSet id_result_set(db_connection->getLastResultSet());
    const unsigned user_count(id_result_set.size());
    while (const DbRow id_row = id_result_set.getNextRow()) {
        const std::string user_id(id_row["id"]);

        const std::string SELECT_SUBSCRIPTION_INFO("SELECT journal_control_number,last_issue_date FROM "
                                                   "ixtheo_journal_subscriptions WHERE id=" + user_id);
        if (unlikely(not db_connection->query(SELECT_SUBSCRIPTION_INFO)))
            Error("Select failed: " + SELECT_SUBSCRIPTION_INFO + " (" + db_connection->getLastErrorMessage() + ")");

        DbResultSet result_set(db_connection->getLastResultSet());
        std::vector<SerialControlNumberAndLastIssueDate> control_numbers_and_last_issue_dates;
        while (const DbRow row = result_set.getNextRow()) {
            control_numbers_and_last_issue_dates.emplace_back(SerialControlNumberAndLastIssueDate(
                row["journal_control_number"], row["last_issue_date"]));
            ++subscription_count;
        }
        ProcessSingleUser(verbose, db_connection, user_id, solr_host_and_port, control_numbers_and_last_issue_dates);
    }

    if (verbose)
        std::cout << "Processed " << user_count << " users and " << subscription_count << " subscriptions.\n";
}

int main(int argc, char **argv) {
    progname = argv[0];

    if (argc < 2)
        Usage();

    bool verbose;
    if (std::strcmp("--verbose", argv[1]) == 0) {
        if (argc < 3)
            Usage();
        verbose = true;
        --argc, ++argv;
    } else
        verbose = false;

    std::string solr_host_and_port;
    if (argc == 2)
        solr_host_and_port = "localhost:8080";
    else if (argc == 3) {
        solr_host_and_port = argv[1];
        --argc, ++argv;
    } else
        Usage();
    const std::string vufind_config_path(argv[1]);

    try {
        std::string mysql_url;
        VuFind::GetMysqlURL(&mysql_url, vufind_config_path);
        DbConnection db_connection(mysql_url);

        ProcessSubscriptions(verbose, &db_connection, solr_host_and_port);
    } catch (const std::exception &x) {
        Error("caught exception: " + std::string(x.what()));
    }
}
<commit_msg>Changed how to run new_journal_alert.<commit_after>/** \file    new_journal_alert.cc
 *  \brief   Detects new journal issues for subscribed users.
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2016, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <sstream>
#include <vector>
#include <cstdlib>
#include <cstring>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include "Compiler.h"
#include "DbConnection.h"
#include "EmailSender.h"
#include "ExecUtil.h"
#include "FileUtil.h"
#include "HtmlUtil.h"
#include "MiscUtil.h"
#include "Solr.h"
#include "StringUtil.h"
#include "UrlUtil.h"
#include "util.h"
#include "VuFind.h"

void Usage() {
    std::cerr << "Usage: " << ::progname << " [--verbose] [solr_host_and_port] hostname\n";
    std::cerr << "  Sends out notification emails for journal subscribers.\n";
    std::cerr << "  Should \"solr_host_and_port\" be missing \"localhost:8080\" will be used.\n";
    std::cerr << "  \"hostname\" should be the symbolic hostname which will be used in constructing\n";
    std::cerr<< "   URL's that a user might see.\n\n";
    std::exit(EXIT_FAILURE);
}

struct SerialControlNumberAndLastIssueDate {
    std::string serial_control_number_;
    std::string last_issue_date_;
    bool changed_;
public:
    SerialControlNumberAndLastIssueDate(const std::string &serial_control_number, const std::string &last_issue_date)
        : serial_control_number_(serial_control_number), last_issue_date_(last_issue_date), changed_(false) { }
    inline void setLastIssueDate(const std::string &new_last_issue_date)
        { last_issue_date_ = new_last_issue_date; changed_ = true; }
    inline bool changed() const { return changed_; }
};

struct NewIssueInfo {
    std::string control_number_;
    std::string journal_title_;
    std::string issue_title_;
    std::string last_issue_date_;
public:
    NewIssueInfo(const std::string &control_number, const std::string &journal_title, const std::string &issue_title)
        : control_number_(control_number), journal_title_(journal_title), issue_title_(issue_title) { }
};

/** \return True if new issues were found, false o/w. */
bool ExtractNewIssueInfos(const std::string &json_document, std::vector<NewIssueInfo> * const new_issue_infos,
                          std::string * const max_last_issue_date)
{
    std::stringstream input(json_document, std::ios_base::in);
    boost::property_tree::ptree property_tree;
    boost::property_tree::json_parser::read_json(input, property_tree);

    bool found_at_least_one_new_issue(false);
    for (const auto &document : property_tree.get_child("response.docs.")) {
        const auto &id(document.second.get<std::string>("id"));
        const auto &issue_title(document.second.get<std::string>("title"));
        const std::string journal_title(document.second.get<std::string>("journal_issue/0", "*No Journal Title*"));
        new_issue_infos->emplace_back(id, journal_title, issue_title);

        const auto &recording_date(document.second.get<std::string>("recording_date"));
        if (recording_date > *max_last_issue_date) {
            *max_last_issue_date = recording_date;
            found_at_least_one_new_issue = true;
        }
    }

    return found_at_least_one_new_issue;
}

bool GetNewIssues(const std::string &solr_host_and_port, const std::string &serial_control_number,
                  std::string last_issue_date, std::vector<NewIssueInfo> * const new_issue_infos,
                  std::string * const max_last_issue_date)
{
    if (not StringUtil::EndsWith(last_issue_date, "Z"))
        last_issue_date += "T00:00:00Z"; // Solr does not support the short form of the ISO 8601 date formats.
    const std::string QUERY("superior_ppn:" + serial_control_number + " AND recording_date:{" + last_issue_date
                            + " TO *}");
    std::string json_result;
    if (unlikely(not Solr::Query(QUERY, "id,title,author,recording_date", &json_result, solr_host_and_port,
                                 /* timeout = */ 5, Solr::JSON)))
        Error("Solr query failed or timed-out: \"" + QUERY + "\".");

    return ExtractNewIssueInfos(json_result, new_issue_infos, max_last_issue_date);
}

void SendNotificationEmail(const std::string &firstname, const std::string &lastname, const std::string &email,
                           const std::string &vufind_host, const std::vector<NewIssueInfo> &new_issue_infos)
{
    const std::string EMAIL_TEMPLATE_PATH("/var/lib/tuelib/subscriptions_email.template");
    std::string email_template;
    if (unlikely(not FileUtil::ReadString(EMAIL_TEMPLATE_PATH, &email_template)))
        Error("can't load email template \"" + EMAIL_TEMPLATE_PATH + "\"!");

    // Process the email template:
    std::map<std::string, std::vector<std::string>> names_to_values_map;
    names_to_values_map["firstname"] = std::vector<std::string>{ firstname };
    names_to_values_map["lastname"] = std::vector<std::string>{ lastname };
    std::vector<std::string> urls, journal_titles, issue_titles;
    for (const auto &new_issue_info : new_issue_infos) {
        urls.emplace_back("https://" + vufind_host + "/Record/" + new_issue_info.control_number_);
        journal_titles.emplace_back(new_issue_info.journal_title_);
        issue_titles.emplace_back(HtmlUtil::HtmlEscape(new_issue_info.issue_title_));
    }
    names_to_values_map["url"]           = urls;
    names_to_values_map["journal_title"] = journal_titles;
    names_to_values_map["issue_title"]   = issue_titles;
    std::istringstream input(email_template);
    std::ostringstream email_contents;
    MiscUtil::ExpandTemplate(input, email_contents, names_to_values_map);

    if (unlikely(not EmailSender::SendEmail("notifications@ixtheo.de", email, "Ixtheo Subscriptions",
                                            email_contents.str(), EmailSender::DO_NOT_SET_PRIORITY,
                                            EmailSender::HTML)))
        Error("failed to send a notification email to \"" + email + "\"!");
}

void ProcessSingleUser(const bool verbose, DbConnection * const db_connection, const std::string &user_id,
                       const std::string &solr_host_and_port, const std::string &hostname,
                       std::vector<SerialControlNumberAndLastIssueDate> &control_numbers_and_last_issue_dates)
{
    const std::string SELECT_USER_ATTRIBUTES("SELECT * FROM user WHERE id=" + user_id);
    if (unlikely(not db_connection->query(SELECT_USER_ATTRIBUTES)))
        Error("Select failed: " + SELECT_USER_ATTRIBUTES + " (" + db_connection->getLastErrorMessage() + ")");
    DbResultSet result_set(db_connection->getLastResultSet());
    if (result_set.empty())
        Error("found no user attributes in table \"user\" for ID \"" + user_id + "\"!");
    if (result_set.size() > 1)
        Error("found multiple user attribute sets in table \"user\" for ID \"" + user_id + "\"!");

    const DbRow row(result_set.getNextRow());
    const std::string username(row["username"]);
    if (verbose)
        std::cerr << "Found " << control_numbers_and_last_issue_dates.size() << " subscriptions for \"" << username
                  << ".\n";

    const std::string firstname(row["firstname"]);
    const std::string lastname(row["lastname"]);
    const std::string email(row["email"]);

    // Collect the dates for new issues.
    std::vector<NewIssueInfo> new_issue_infos;
    for (auto &control_number_and_last_issue_date : control_numbers_and_last_issue_dates) {
        std::string max_last_issue_date(control_number_and_last_issue_date.last_issue_date_);
        if (GetNewIssues(solr_host_and_port, control_number_and_last_issue_date.serial_control_number_,
                         control_number_and_last_issue_date.last_issue_date_, &new_issue_infos, &max_last_issue_date))
            control_number_and_last_issue_date.setLastIssueDate(max_last_issue_date);
    }
    if (verbose)
        std::cerr << "Found " << new_issue_infos.size() << " new issues for " << " \"" << username << "\".\n";

    if (not new_issue_infos.empty())
        SendNotificationEmail(firstname, lastname, email, hostname, new_issue_infos);

    // Update the database with the new last issue dates.
    for (const auto &control_number_and_last_issue_date : control_numbers_and_last_issue_dates) {
        if (not control_number_and_last_issue_date.changed())
            continue;

        const std::string REPLACE_STMT("REPLACE INTO ixtheo_journal_subscriptions SET id=" + user_id
                                       + ",last_issue_date='" + control_number_and_last_issue_date.last_issue_date_
                                       + "',journal_control_number='"
                                       + control_number_and_last_issue_date.serial_control_number_ + "'");
        if (unlikely(not db_connection->query(REPLACE_STMT)))
            Error("Replace failed: " + REPLACE_STMT + " (" + db_connection->getLastErrorMessage() + ")");
    }
}

void ProcessSubscriptions(const bool verbose, DbConnection * const db_connection,
                          const std::string &solr_host_and_port, const std::string &hostname)
{
    const std::string SELECT_IDS_STMT("SELECT DISTINCT id FROM ixtheo_journal_subscriptions");
    if (unlikely(not db_connection->query(SELECT_IDS_STMT)))
        Error("Select failed: " + SELECT_IDS_STMT + " (" + db_connection->getLastErrorMessage() + ")");

    unsigned subscription_count(0);
    DbResultSet id_result_set(db_connection->getLastResultSet());
    const unsigned user_count(id_result_set.size());
    while (const DbRow id_row = id_result_set.getNextRow()) {
        const std::string user_id(id_row["id"]);

        const std::string SELECT_SUBSCRIPTION_INFO("SELECT journal_control_number,last_issue_date FROM "
                                                   "ixtheo_journal_subscriptions WHERE id=" + user_id);
        if (unlikely(not db_connection->query(SELECT_SUBSCRIPTION_INFO)))
            Error("Select failed: " + SELECT_SUBSCRIPTION_INFO + " (" + db_connection->getLastErrorMessage() + ")");

        DbResultSet result_set(db_connection->getLastResultSet());
        std::vector<SerialControlNumberAndLastIssueDate> control_numbers_and_last_issue_dates;
        while (const DbRow row = result_set.getNextRow()) {
            control_numbers_and_last_issue_dates.emplace_back(SerialControlNumberAndLastIssueDate(
                row["journal_control_number"], row["last_issue_date"]));
            ++subscription_count;
        }
        ProcessSingleUser(verbose, db_connection, hostname, user_id, solr_host_and_port,
                          control_numbers_and_last_issue_dates);
    }

    if (verbose)
        std::cout << "Processed " << user_count << " users and " << subscription_count << " subscriptions.\n";
}

int main(int argc, char **argv) {
    progname = argv[0];

    if (argc < 2)
        Usage();

    bool verbose;
    if (std::strcmp("--verbose", argv[1]) == 0) {
        if (argc < 3)
            Usage();
        verbose = true;
        --argc, ++argv;
    } else
        verbose = false;

    std::string solr_host_and_port;
    if (argc == 2)
        solr_host_and_port = "localhost:8080";
    else if (argc == 3) {
        solr_host_and_port = argv[1];
        --argc, ++argv;
    } else
        Usage();
    const std::string hostname(argv[1]);

    try {
        std::string mysql_url;
        VuFind::GetMysqlURL(&mysql_url);
        DbConnection db_connection(mysql_url);

        ProcessSubscriptions(verbose, &db_connection, solr_host_and_port, hostname);
    } catch (const std::exception &x) {
        Error("caught exception: " + std::string(x.what()));
    }
}
<|endoftext|>