<commit_before>/**
 * @file
 * @brief Implementation of Geant4 deposition module
 * @remarks Based on code from Mathieu Benoit
 * @copyright Copyright (c) 2017 CERN and the Allpix Squared authors.
 * This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
 * In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
 * Intergovernmental Organization or submit itself to any jurisdiction.
 */

#include "DepositionGeant4Module.hpp"

#include <limits>
#include <string>
#include <utility>

#include <G4EmParameters.hh>
#include <G4HadronicProcessStore.hh>
#include <G4LogicalVolume.hh>
#include <G4PhysListFactory.hh>
#include <G4RunManager.hh>
#include <G4StepLimiterPhysics.hh>
#include <G4UImanager.hh>
#include <G4UserLimits.hh>

#include "core/config/exceptions.h"
#include "core/geometry/GeometryManager.hpp"
#include "core/module/exceptions.h"
#include "core/utils/log.h"
#include "objects/DepositedCharge.hpp"
#include "tools/ROOT.h"
#include "tools/geant4.h"

#include "GeneratorActionG4.hpp"
#include "SensitiveDetectorActionG4.hpp"

#define G4_NUM_SEEDS 10

using namespace allpix;

/**
 * Includes the particle source point to the geometry using \ref GeometryManager::addPoint.
 */
DepositionGeant4Module::DepositionGeant4Module(Configuration config, Messenger* messenger, GeometryManager* geo_manager)
    : Module(config), config_(std::move(config)), messenger_(messenger), geo_manager_(geo_manager), last_event_num_(1),
      run_manager_g4_(nullptr) {
    // Create user limits for maximum step length in the sensor
    user_limits_ = std::make_unique<G4UserLimits>(config_.get<double>("max_step_length", Units::get(1.0, "um")));

    // Set default physics list
    config_.setDefault("physics_list", "FTFP_BERT_LIV");

    // Add the particle source position to the geometry
    geo_manager_->addPoint(config_.get<ROOT::Math::XYZPoint>("beam_position"));
}

/**
 * Module depends on \ref GeometryBuilderGeant4Module loaded first, because it owns the pointer to the Geant4 run manager.
 */
void DepositionGeant4Module::init() {
    // Load the G4 run manager (which is owned by the geometry builder)
    run_manager_g4_ = G4RunManager::GetRunManager();
    if(run_manager_g4_ == nullptr) {
        throw ModuleError("Cannot deposit charges using Geant4 without a Geant4 geometry builder");
    }

    // Suppress all output from G4
    SUPPRESS_STREAM(G4cout);

    // Get UI manager for sending commands
    G4UImanager* ui_g4 = G4UImanager::GetUIpointer();

    // Apply optional PAI model
    if(config_.get<bool>("enable_pai", false)) {
        LOG(TRACE) << "Enabling PAI model on all detectors";
        G4EmParameters::Instance();

        for(auto& detector : geo_manager_->getDetectors()) {
            // Get logical volume
            auto logical_volume = detector->getExternalObject<G4LogicalVolume>("sensor_log");
            if(logical_volume == nullptr) {
                throw ModuleError("Detector " + detector->getName() + " has no sensitive device (broken Geant4 geometry)");
            }
            // Create region
            G4Region* region = new G4Region(detector->getName() + "_sensor_region");
            region->AddRootLogicalVolume(logical_volume.get());

            auto pai_model = config_.get<std::string>("pai_model", "pai");
            auto lcase_model = pai_model;
            std::transform(lcase_model.begin(), lcase_model.end(), lcase_model.begin(), ::tolower);
            if(lcase_model == "pai") {
                pai_model = "PAI";
            } else if(lcase_model == "paiphoton") {
                pai_model = "PAIphoton";
            } else {
                throw InvalidValueError(config_, "pai_model", "model has to be either 'pai' or 'paiphoton'");
            }

            ui_g4->ApplyCommand("/process/em/AddPAIRegion all " + region->GetName() + " " + pai_model);
        }
    }

    // Find the physics list
    G4PhysListFactory physListFactory;
    G4VModularPhysicsList* physicsList = physListFactory.GetReferencePhysList(config_.get<std::string>("physics_list"));
    if(physicsList == nullptr) {
        std::string message = "specified physics list does not exists";
        std::vector<G4String> base_lists = physListFactory.AvailablePhysLists();
        message += " (available base lists are ";
        for(auto& base_list : base_lists) {
            message += base_list;
            message += ", ";
        }
        message = message.substr(0, message.size() - 2);
        message += " with optional suffixes for electromagnetic lists ";
        std::vector<G4String> em_lists = physListFactory.AvailablePhysListsEM();
        for(auto& em_list : em_lists) {
            if(em_list.empty()) {
                continue;
            }
            message += em_list;
            message += ", ";
        }
        message = message.substr(0, message.size() - 2);
        message += ")";

        throw InvalidValueError(config_, "physics_list", message);
    } else {
        LOG(INFO) << "Using G4 physics list \"" << config_.get<std::string>("physics_list") << "\"";
    }
    // Register a step limiter (uses the user limits defined earlier)
    physicsList->RegisterPhysics(new G4StepLimiterPhysics());
    // Initialize the physics list
    LOG(TRACE) << "Initializing physics processes";
    run_manager_g4_->SetUserInitialization(physicsList);
    run_manager_g4_->InitializePhysics();

    // Initialize the full run manager to ensure correct state flags
    run_manager_g4_->Initialize();

    // Build particle generator
    LOG(TRACE) << "Constructing particle source";
    auto generator = new GeneratorActionG4(config_);
    run_manager_g4_->SetUserAction(generator);

    // Get the creation energy for charge (default is silicon electron hole pair energy)
    auto charge_creation_energy = config_.get<double>("charge_creation_energy", Units::get(3.64, "eV"));

    // Loop through all detectors and set the sensitive detector action that handles the particle passage
    bool useful_deposition = false;
    for(auto& detector : geo_manager_->getDetectors()) {
        // Do not add sensitive detector for detectors that have no listeners for the deposited charges
        // FIXME Probably the MCParticle has to be checked as well
        if(!messenger_->hasReceiver(this,
                                    std::make_shared<DepositedChargeMessage>(std::vector<DepositedCharge>(), detector))) {
            LOG(INFO) << "Not depositing charges in " << detector->getName()
                      << " because there is no listener for its output";
            continue;
        }
        useful_deposition = true;

        // Get model of the sensitive device
        auto sensitive_detector_action = new SensitiveDetectorActionG4(this, detector, messenger_, charge_creation_energy);
        auto logical_volume = detector->getExternalObject<G4LogicalVolume>("sensor_log");
        if(logical_volume == nullptr) {
            throw ModuleError("Detector " + detector->getName() + " has no sensitive device (broken Geant4 geometry)");
        }

        // Apply the user limits to this element
        logical_volume->SetUserLimits(user_limits_.get());

        // Add the sensitive detector action
        logical_volume->SetSensitiveDetector(sensitive_detector_action);
        sensors_.push_back(sensitive_detector_action);
    }

    if(!useful_deposition) {
        LOG(ERROR) << "Not a single listener for deposited charges, module is useless!";
    }

    // Disable verbose messages from processes
    ui_g4->ApplyCommand("/process/verbose 0");
    ui_g4->ApplyCommand("/process/em/verbose 0");
    ui_g4->ApplyCommand("/process/eLoss/verbose 0");
    G4HadronicProcessStore::Instance()->SetVerbose(0);

    // Set the random seed for Geant4 generation
    // NOTE Assumes this is the only Geant4 module using random numbers
    std::string seed_command = "/random/setSeeds ";
    for(int i = 0; i < G4_NUM_SEEDS; ++i) {
        seed_command += std::to_string(static_cast<uint32_t>(getRandomSeed() % INT_MAX));
        if(i != G4_NUM_SEEDS - 1) {
            seed_command += " ";
        }
    }
    ui_g4->ApplyCommand(seed_command);

    // Release the output stream
    RELEASE_STREAM(G4cout);
}

void DepositionGeant4Module::run(unsigned int event_num) {
    // Suppress output stream if not in debugging mode
    IFLOG(DEBUG);
    else {
        SUPPRESS_STREAM(G4cout);
    }

    // Start a single event from the beam
    LOG(TRACE) << "Enabling beam";
    run_manager_g4_->BeamOn(static_cast<int>(config_.get<unsigned int>("number_of_particles", 1)));
    last_event_num_ = event_num;

    // Release the stream (if it was suspended)
    RELEASE_STREAM(G4cout);

    // Dispatch the necessary messages
    for(auto& sensor : sensors_) {
        sensor->dispatchMessages();
    }
}

void DepositionGeant4Module::finalize() {
    size_t total_charges = 0;
    for(auto& sensor : sensors_) {
        total_charges += sensor->getTotalDepositedCharge();
    }

    // Print summary or warns if module did not output any charges
    if(!sensors_.empty() && total_charges > 0 && last_event_num_ > 0) {
        size_t average_charge = total_charges / sensors_.size() / last_event_num_;
        LOG(INFO) << "Deposited total of " << total_charges << " charges in " << sensors_.size() << " sensor(s) (average of "
                  << average_charge << " per sensor for every event)";
    } else {
        LOG(WARNING) << "No charges deposited";
    }
}
<commit_msg>Implement production cut in DepositionGeant4<commit_after>/**
 * @file
 * @brief Implementation of Geant4 deposition module
 * @remarks Based on code from Mathieu Benoit
 * @copyright Copyright (c) 2017 CERN and the Allpix Squared authors.
 * This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
 * In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
 * Intergovernmental Organization or submit itself to any jurisdiction.
 */

#include "DepositionGeant4Module.hpp"

#include <limits>
#include <string>
#include <utility>

#include <G4EmParameters.hh>
#include <G4HadronicProcessStore.hh>
#include <G4LogicalVolume.hh>
#include <G4PhysListFactory.hh>
#include <G4RunManager.hh>
#include <G4StepLimiterPhysics.hh>
#include <G4UImanager.hh>
#include <G4UserLimits.hh>

#include "core/config/exceptions.h"
#include "core/geometry/GeometryManager.hpp"
#include "core/module/exceptions.h"
#include "core/utils/log.h"
#include "objects/DepositedCharge.hpp"
#include "tools/ROOT.h"
#include "tools/geant4.h"

#include "GeneratorActionG4.hpp"
#include "SensitiveDetectorActionG4.hpp"

#define G4_NUM_SEEDS 10

using namespace allpix;

/**
 * Includes the particle source point to the geometry using \ref GeometryManager::addPoint.
 */
DepositionGeant4Module::DepositionGeant4Module(Configuration config, Messenger* messenger, GeometryManager* geo_manager)
    : Module(config), config_(std::move(config)), messenger_(messenger), geo_manager_(geo_manager), last_event_num_(1),
      run_manager_g4_(nullptr) {
    // Create user limits for maximum step length in the sensor
    user_limits_ = std::make_unique<G4UserLimits>(config_.get<double>("max_step_length", Units::get(1.0, "um")));

    // Set default physics list
    config_.setDefault("physics_list", "FTFP_BERT_LIV");

    // Add the particle source position to the geometry
    geo_manager_->addPoint(config_.get<ROOT::Math::XYZPoint>("beam_position"));
}

/**
 * Module depends on \ref GeometryBuilderGeant4Module loaded first, because it owns the pointer to the Geant4 run manager.
 */
void DepositionGeant4Module::init() {
    // Load the G4 run manager (which is owned by the geometry builder)
    run_manager_g4_ = G4RunManager::GetRunManager();
    if(run_manager_g4_ == nullptr) {
        throw ModuleError("Cannot deposit charges using Geant4 without a Geant4 geometry builder");
    }

    // Suppress all output from G4
    SUPPRESS_STREAM(G4cout);

    // Get UI manager for sending commands
    G4UImanager* ui_g4 = G4UImanager::GetUIpointer();

    // Apply optional PAI model
    if(config_.get<bool>("enable_pai", false)) {
        LOG(TRACE) << "Enabling PAI model on all detectors";
        G4EmParameters::Instance();

        for(auto& detector : geo_manager_->getDetectors()) {
            // Get logical volume
            auto logical_volume = detector->getExternalObject<G4LogicalVolume>("sensor_log");
            if(logical_volume == nullptr) {
                throw ModuleError("Detector " + detector->getName() + " has no sensitive device (broken Geant4 geometry)");
            }
            // Create region
            G4Region* region = new G4Region(detector->getName() + "_sensor_region");
            region->AddRootLogicalVolume(logical_volume.get());

            auto pai_model = config_.get<std::string>("pai_model", "pai");
            auto lcase_model = pai_model;
            std::transform(lcase_model.begin(), lcase_model.end(), lcase_model.begin(), ::tolower);
            if(lcase_model == "pai") {
                pai_model = "PAI";
            } else if(lcase_model == "paiphoton") {
                pai_model = "PAIphoton";
            } else {
                throw InvalidValueError(config_, "pai_model", "model has to be either 'pai' or 'paiphoton'");
            }

            ui_g4->ApplyCommand("/process/em/AddPAIRegion all " + region->GetName() + " " + pai_model);
        }
    }

    // Find the physics list
    G4PhysListFactory physListFactory;
    G4VModularPhysicsList* physicsList = physListFactory.GetReferencePhysList(config_.get<std::string>("physics_list"));
    if(physicsList == nullptr) {
        std::string message = "specified physics list does not exists";
        std::vector<G4String> base_lists = physListFactory.AvailablePhysLists();
        message += " (available base lists are ";
        for(auto& base_list : base_lists) {
            message += base_list;
            message += ", ";
        }
        message = message.substr(0, message.size() - 2);
        message += " with optional suffixes for electromagnetic lists ";
        std::vector<G4String> em_lists = physListFactory.AvailablePhysListsEM();
        for(auto& em_list : em_lists) {
            if(em_list.empty()) {
                continue;
            }
            message += em_list;
            message += ", ";
        }
        message = message.substr(0, message.size() - 2);
        message += ")";

        throw InvalidValueError(config_, "physics_list", message);
    } else {
        LOG(INFO) << "Using G4 physics list \"" << config_.get<std::string>("physics_list") << "\"";
    }
    // Register a step limiter (uses the user limits defined earlier)
    physicsList->RegisterPhysics(new G4StepLimiterPhysics());

    //Define the production cut as one fifth of the minimum size (thickness, pitch) among the detectors
    double min_size = std::numeric_limits<double>::max();
    for(auto& detector : geo_manager_->getDetectors()) {
        auto model = detector->getModel();
        min_size = std::min(std::min(std::min(min_size, model->getPixelSize().x()), model->getPixelSize().y()), model->getSensorSize().z());
        }
    auto production_cut = std::to_string(min_size/5);
    ui_g4->ApplyCommand("/run/setCut " + production_cut);

    // Initialize the physics list
    LOG(TRACE) << "Initializing physics processes";
    run_manager_g4_->SetUserInitialization(physicsList);
    run_manager_g4_->InitializePhysics();

    // Initialize the full run manager to ensure correct state flags
    run_manager_g4_->Initialize();

    // Build particle generator
    LOG(TRACE) << "Constructing particle source";
    auto generator = new GeneratorActionG4(config_);
    run_manager_g4_->SetUserAction(generator);

    // Get the creation energy for charge (default is silicon electron hole pair energy)
    auto charge_creation_energy = config_.get<double>("charge_creation_energy", Units::get(3.64, "eV"));

    // Loop through all detectors and set the sensitive detector action that handles the particle passage
    bool useful_deposition = false;
    for(auto& detector : geo_manager_->getDetectors()) {
        // Do not add sensitive detector for detectors that have no listeners for the deposited charges
        // FIXME Probably the MCParticle has to be checked as well
        if(!messenger_->hasReceiver(this,
                                    std::make_shared<DepositedChargeMessage>(std::vector<DepositedCharge>(), detector))) {
            LOG(INFO) << "Not depositing charges in " << detector->getName()
                      << " because there is no listener for its output";
            continue;
        }
        useful_deposition = true;

        // Get model of the sensitive device
        auto sensitive_detector_action = new SensitiveDetectorActionG4(this, detector, messenger_, charge_creation_energy);
        auto logical_volume = detector->getExternalObject<G4LogicalVolume>("sensor_log");
        if(logical_volume == nullptr) {
            throw ModuleError("Detector " + detector->getName() + " has no sensitive device (broken Geant4 geometry)");
        }

        // Apply the user limits to this element
        logical_volume->SetUserLimits(user_limits_.get());

        // Add the sensitive detector action
        logical_volume->SetSensitiveDetector(sensitive_detector_action);
        sensors_.push_back(sensitive_detector_action);
    }

    if(!useful_deposition) {
        LOG(ERROR) << "Not a single listener for deposited charges, module is useless!";
    }

    // Disable verbose messages from processes
    ui_g4->ApplyCommand("/process/verbose 0");
    ui_g4->ApplyCommand("/process/em/verbose 0");
    ui_g4->ApplyCommand("/process/eLoss/verbose 0");
    G4HadronicProcessStore::Instance()->SetVerbose(0);

    // Set the random seed for Geant4 generation
    // NOTE Assumes this is the only Geant4 module using random numbers
    std::string seed_command = "/random/setSeeds ";
    for(int i = 0; i < G4_NUM_SEEDS; ++i) {
        seed_command += std::to_string(static_cast<uint32_t>(getRandomSeed() % INT_MAX));
        if(i != G4_NUM_SEEDS - 1) {
            seed_command += " ";
        }
    }
    ui_g4->ApplyCommand(seed_command);

    // Release the output stream
    RELEASE_STREAM(G4cout);
}

void DepositionGeant4Module::run(unsigned int event_num) {
    // Suppress output stream if not in debugging mode
    IFLOG(DEBUG);
    else {
        SUPPRESS_STREAM(G4cout);
    }

    // Start a single event from the beam
    LOG(TRACE) << "Enabling beam";
    run_manager_g4_->BeamOn(static_cast<int>(config_.get<unsigned int>("number_of_particles", 1)));
    last_event_num_ = event_num;

    // Release the stream (if it was suspended)
    RELEASE_STREAM(G4cout);

    // Dispatch the necessary messages
    for(auto& sensor : sensors_) {
        sensor->dispatchMessages();
    }
}

void DepositionGeant4Module::finalize() {
    size_t total_charges = 0;
    for(auto& sensor : sensors_) {
        total_charges += sensor->getTotalDepositedCharge();
    }

    // Print summary or warns if module did not output any charges
    if(!sensors_.empty() && total_charges > 0 && last_event_num_ > 0) {
        size_t average_charge = total_charges / sensors_.size() / last_event_num_;
        LOG(INFO) << "Deposited total of " << total_charges << " charges in " << sensors_.size() << " sensor(s) (average of "
                  << average_charge << " per sensor for every event)";
    } else {
        LOG(WARNING) << "No charges deposited";
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2012, Steinwurf ApS
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//     * Neither the name of Steinwurf ApS nor the
//       names of its contributors may be used to endorse or promote products
//       derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL Steinwurf ApS BE LIABLE FOR ANY
// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "file_input_stream.hpp"
#include "error.hpp"

#include <fstream>
#include <cassert>

namespace sak
{

    file_input_stream::file_input_stream()
        : m_filesize(0)
    {
    }

    file_input_stream::file_input_stream(const std::string& filename)
        : m_filesize(0)
    {
        open(filename);
    }

    void file_input_stream::open(const std::string& filename)
    {
        assert(!m_file.is_open());

        boost::system::error_code ec;
        open(filename, ec);

        // If an error throw
        error::throw_error(ec);
    }

    void file_input_stream::open(const std::string& filename,
                                 boost::system::error_code& ec)
    {
        assert(!m_file.is_open());

        m_file.open(filename.c_str(),
                    std::ios::in | std::ios::binary);

        if (!m_file.is_open())
        {
            ec = error::make_error_code(error::failed_open_file);
            return;
        }

        m_file.seekg(0, std::ios::end);
        assert(m_file);

        m_filesize = read_position();
        m_file.seekg(0, std::ios::beg);
        assert(m_file);
    }

    void file_input_stream::seek(uint32_t pos)
    {
        assert(m_file.is_open());
        m_file.seekg(pos, std::ios::beg);
        assert(m_file);

        // if(m_file.fail())
        // {
        //     ec = error::make_error_code(error::failed_open_file);
        //     return;
        // }
    }

    uint32_t file_input_stream::read_position()
    {
        assert(m_file.is_open());

        std::cout << "EOF " << m_file.eof() << std::endl;

        std::streamoff pos = m_file.tellg();
        std::cout << "tellg = " << (int32_t) pos << std::endl;

        assert(pos >= 0);

        return static_cast<uint32_t>(pos);
    }

    void file_input_stream::read(uint8_t* buffer, uint32_t bytes)
    {
        assert(m_file.is_open());
        std::cout << "file_input_stream::read " << bytes << std::endl;

        m_file.read(reinterpret_cast<char*>(buffer), bytes);

        assert(bytes == static_cast<uint32_t>(m_file.gcount()));
    }

    uint32_t file_input_stream::bytes_available()
    {
        assert(m_file.is_open());
        std::cout << "file_input_stream::bytes_available " << std::endl;

        uint32_t pos = read_position();
        assert(pos <= m_filesize);

        return m_filesize - pos;
    }

    uint32_t file_input_stream::size()
    {
        assert(m_file.is_open());
        return m_filesize;
    }

}
<commit_msg>Check for EOF when checking read_position()<commit_after>// Copyright (c) 2012, Steinwurf ApS
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//     * Neither the name of Steinwurf ApS nor the
//       names of its contributors may be used to endorse or promote products
//       derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL Steinwurf ApS BE LIABLE FOR ANY
// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "file_input_stream.hpp"
#include "error.hpp"

#include <fstream>
#include <cassert>

namespace sak
{

    file_input_stream::file_input_stream()
        : m_filesize(0)
    {
    }

    file_input_stream::file_input_stream(const std::string& filename)
        : m_filesize(0)
    {
        open(filename);
    }

    void file_input_stream::open(const std::string& filename)
    {
        assert(!m_file.is_open());

        boost::system::error_code ec;
        open(filename, ec);

        // If an error throw
        error::throw_error(ec);
    }

    void file_input_stream::open(const std::string& filename,
                                 boost::system::error_code& ec)
    {
        assert(!m_file.is_open());

        m_file.open(filename.c_str(),
                    std::ios::in | std::ios::binary);

        if (!m_file.is_open())
        {
            ec = error::make_error_code(error::failed_open_file);
            return;
        }

        m_file.seekg(0, std::ios::end);
        assert(m_file);

        // We cannot use the read_position function here due to a
        // problem on the iOS platform described in the read_position
        // function.
        auto pos = m_file.tellg();
        assert(pos >= 0);

        m_filesize = (uint32_t) pos;

        m_file.seekg(0, std::ios::beg);
        assert(m_file);
    }

    void file_input_stream::seek(uint32_t pos)
    {
        assert(m_file.is_open());
        m_file.seekg(pos, std::ios::beg);
        assert(m_file);

        // if(m_file.fail())
        // {
        //     ec = error::make_error_code(error::failed_open_file);
        //     return;
        // }
    }

    uint32_t file_input_stream::read_position()
    {
        assert(m_file.is_open());

        // Work around for problem on iOS where tellg returned -1 when
        // reading the last byte. However the EOF flag was correctly
        // set. So here we check for EOF if true we set the
        // read_position = m_file_size

        if(m_file.eof())
        {
            return m_file_size;
        }
        else
        {
            std::streamoff pos = m_file.tellg();
            std::cout << "tellg = " << (int32_t) pos << std::endl;

            assert(pos >= 0);

            return static_cast<uint32_t>(pos);
        }
    }

    void file_input_stream::read(uint8_t* buffer, uint32_t bytes)
    {
        assert(m_file.is_open());
        std::cout << "file_input_stream::read " << bytes << std::endl;

        m_file.read(reinterpret_cast<char*>(buffer), bytes);

        assert(bytes == static_cast<uint32_t>(m_file.gcount()));
    }

    uint32_t file_input_stream::bytes_available()
    {
        assert(m_file.is_open());
        std::cout << "file_input_stream::bytes_available " << std::endl;

        uint32_t pos = read_position();
        assert(pos <= m_filesize);

        return m_filesize - pos;
    }

    uint32_t file_input_stream::size()
    {
        assert(m_file.is_open());
        return m_filesize;
    }

}
<|endoftext|>


<commit_before>#include "simulator.hpp"
#include <float.h>

void SMSimulator::init(itpp::bvec _inputBits) {
  inputBits = _inputBits;

  Nt = itpp::length(inputBits);

  ofs << "SNR[dB],BER" << std::endl;

  m = log2(nC * nTx);

  if (Nt % m != 0) {
    inputBits = itpp::concat(inputBits, itpp::zeros_b(Nt % m));
    Nt = itpp::length(inputBits);  // Refresh
  }

  Nu = Nt;
  /* !!!Warning!!! : nAntenna must be modified */
  nAntenna = itpp::length(itpp::dec2bin(nTx)) - 1;
  nSymbols = nC;
  nSymbin = itpp::length(itpp::dec2bin(nSymbols)) - 1;

  modulator.set_M(nSymbols);

  Nctx = Nu;
  Nvec = Nctx / m;
  Nbitspvec = m;

  std::cout << m << " " << Nu << " " << Nctx << " " << Nvec << " " << Nbitspvec << std::endl;

  return;
}

void SMxSimulator::init(itpp::bvec _inputBits) {
  inputBits = _inputBits;

  Nt = itpp::length(inputBits);

  ofs << "SNR[dB],BER" << std::endl;

  m = log2(nC) * nTx;

  if (Nt % m != 0) {
    inputBits = itpp::concat(inputBits, itpp::zeros_b(Nt % m));
    Nt = itpp::length(inputBits);  // Refresh
  }

  Nu = Nt;

  modulator.set_M(nTx, nC);

  Nctx = Nu;
  Nvec = Nctx / m;
  Nbitspvec = m;

  std::cout << m << " " << Nu << " " << Nctx << " " << Nvec << " " << Nbitspvec << std::endl;

  return;
}

itpp::cvec SMSimulator::select_antenna(itpp::bvec bits) {
  itpp::cvec s = itpp::zeros_c(nTx); // アンテナ数
  int Nbits = itpp::length(bits);
  int selected_antenna = itpp::bin2dec(bits(0, nAntenna - 1));

  std::complex<double> symbol = modulator.get_symbols()(itpp::bin2dec(bits(nAntenna, Nbits - 1)));

  if (nAntenna == 0) {
    s(0) = symbol;
  } else {
    s(selected_antenna) = symbol;
  }

  return s;
}

itpp::vec SMSimulator::demodulate_bits(itpp::cvec Y, itpp::cmat H, itpp::cvec S, itpp::Array<itpp::cvec> samples) {
  int i = 0, argmax_P;
  const double coef = pow(itpp::pi, -nRx);

  itpp::vec P(nTx * nSymbols);
  itpp::vec j_q_pair(2);

  for (int j = 0; j < nTx; j++) {
    for (int q = 0; q < nSymbols; q++) {
      if (nTx == 1) {
        P(i++) = -itpp::norm(Y - H * samples(nSymbols * j + q));
      } else {
        P(i++) = coef * exp(-itpp::norm(Y - H * samples(nSymbols * j + q), "fro"));
      }
    }
  }

  argmax_P = itpp::max_index(P);

  j_q_pair(0) = argmax_P / nSymbols;
  j_q_pair(1) = argmax_P % nSymbols;

  return j_q_pair;
}

itpp::Array<itpp::cvec> SMSimulator::get_samples() {
  itpp::cvec symbols = modulator.get_symbols();
  itpp::Array<itpp::cvec> samples(nSymbols * nTx);
  itpp::cvec tmp;

  for (int i = 0; i < nTx; i++) {
    for (int j = 0; j < nSymbols; j++) {
      tmp = itpp::zeros_c(nTx);
      tmp(i) = symbols(j);

      samples(nSymbols * i + j) = tmp;
    }
  }

  return samples;
}

itpp::bvec SMSimulator::simulate(int TotalNumSimulate) {
  std::cout.setf(std::ios::fixed, std::ios::floatfield);
  std::cout.setf(std::ios::showpoint);
  //std::cout.setf(std::ios::scientific, std::ios::floatfield);
  std::cout.precision(10);

  for (int nsnr = 0; nsnr <= TotalNumSimulate; nsnr++) {
    itpp::cvec Y;
    itpp::cmat H;
    itpp::cvec S;
    itpp::cvec e;
    itpp::Array<itpp::cvec> Samples = SMSimulator::get_samples();

    const double sigma2 = itpp::inv_dB(-nsnr);

    itpp::BERC bercu;

    itpp::bvec bitstmp;

    itpp::vec j_q_pair;
    itpp::bvec transmittmp;
    itpp::bvec receivetmp;

    itpp::bvec transmitted(Nctx);
    itpp::bvec received(Nctx);

    for (int k = 0; k < Nvec; k++) {
      H = itpp::randn_c(nRx, nTx);

      bitstmp = inputBits(k * Nbitspvec, (k + 1) * Nbitspvec - 1);

      S = SMSimulator::select_antenna(bitstmp);

      e = sqrt(sigma2) * itpp::randn_c(nRx);

      Y = H * S + e;

      j_q_pair = SMSimulator::demodulate_bits(Y, H, S, Samples);

      transmittmp = bitstmp;
      receivetmp = itpp::concat(itpp::dec2bin(nAntenna, static_cast<int>(j_q_pair(0))),
                                itpp::dec2bin(nSymbin, static_cast<int>(j_q_pair(1)))
                                );

      transmitted.set_subvector(k * Nbitspvec, transmittmp);
      received.set_subvector(k * Nbitspvec, receivetmp);
    }

    bercu.count(transmitted(0, Nu - 1), received(0, Nu - 1));

    std::cout << "-----------------------------------------------------" << std::endl;
    std::cout << "Eb/N0: " << nsnr << " dB. Simulated " << Nctx << " bits." << std::endl;
    //std::cout << " Uncoded BER: " << error_rate << " (ML)" << std::endl;
    std::cout << " Uncoded BER: " << bercu.get_errorrate() << " (ML)" << std::endl;
    ofs << std::dec << nsnr << "," << std::scientific << bercu.get_errorrate() << std::endl;

    // Debug
    return received;
  }

  return itpp::bvec(1);
}

itpp::bvec SMxSimulator::simulate(int TotalNumSimulate) {
  const int Nmethods = 1;

  std::cout.setf(std::ios::fixed, std::ios::floatfield);
  std::cout.setf(std::ios::showpoint);
  std::cout.precision(10);

  for (int nsnr = 0; nsnr <= TotalNumSimulate; nsnr++) {
    itpp::Array<itpp::cvec> Y(Nvec);
    itpp::Array<itpp::cmat> H(Nvec + 1);
    itpp::cvec e;

    const double sigma2 = itpp::inv_dB(-nsnr);

    itpp::BERC bercu;

    itpp::bvec bitstmp;

    itpp::bvec received(Nu);

    for (int k = 0; k < Nvec; k++) {
      H(k) = itpp::randn_c(nRx, nTx);

      bitstmp = inputBits(k * Nbitspvec, (k + 1) * Nbitspvec - 1);

      e = sqrt(sigma2) * itpp::randn_c(nRx);

      //std::cout << length(bitstmp) << " " << sum(modulator.get_k()) << "\n";
      itpp::cvec x = modulator.modulate_bits(bitstmp);

      Y(k) = H(k) * x + e;

      //std::cout << modulator(Y(k)) << std::endl;
      /*
      received.set_subvector(k * Nbitspvec,
                             modulator.get_symbols()(0)(itpp::bin2dec(Y(k)))
                             );
      */
    }

    // -- demodulate --
    itpp::Array<itpp::QLLRvec> LLRin(Nmethods);
    for (int i = 0; i < Nmethods; i++) {
      LLRin(i) = itpp::zeros_i(Nctx);
    }

    itpp::QLLRvec llr_apr = itpp::zeros_i(Nbitspvec);
    itpp::QLLRvec llr_apost = itpp::zeros_i(Nbitspvec);

    for (int k = 0; k < Nvec; k++) {
      // ML demodulation
      modulator.demodulate_soft_bits(Y(k), H(k), sigma2, llr_apr, llr_apost);
      LLRin(0).set_subvector(k * Nbitspvec, llr_apost);
    }

    // -- decode and count errors --
    for (int i = 0; i < Nmethods; i++) {
      bercu.count(inputBits(0, Nu - 1), LLRin(0)(0, Nu - 1) < 0);
    }

    // -- received bits --
    for (int i = 0; i < Nu; i++) {
      received(i) = LLRin(0)(i) < 0;
    }

    //bercu.count(transmitted(0, Nu - 1), received(0, Nu - 1));

    std::cout << "-----------------------------------------------------" << std::endl;
    std::cout << "Eb/N0: " << nsnr << " dB. Simulated " << Nctx << " bits." << std::endl;
    //std::cout << " Uncoded BER: " << error_rate << " (ML)" << std::endl;
    std::cout << " Uncoded BER: " << bercu.get_errorrate() << " (ML)" << std::endl;
    ofs << std::dec << nsnr << "," << std::scientific << bercu.get_errorrate() << std::endl;

    // Debug
    return received;
  }

  return itpp::bvec(1);
}
<commit_msg>Modify some indents<commit_after>#include "simulator.hpp"
#include <float.h>

void SMSimulator::init(itpp::bvec _inputBits) {
  inputBits = _inputBits;

  Nt = itpp::length(inputBits);

  ofs << "SNR[dB],BER" << std::endl;

  m = log2(nC * nTx);

  if (Nt % m != 0) {
    inputBits = itpp::concat(inputBits, itpp::zeros_b(Nt % m));
    Nt = itpp::length(inputBits);  // Refresh
  }

  Nu = Nt;
  nAntenna = itpp::length(itpp::dec2bin(nTx)) - 1;
  nSymbols = nC;
  nSymbin = itpp::length(itpp::dec2bin(nSymbols)) - 1;

  modulator.set_M(nSymbols);

  Nctx = Nu;
  Nvec = Nctx / m;
  Nbitspvec = m;

  std::cout << m << " " << Nu << " " << Nctx << " " << Nvec << " " << Nbitspvec << std::endl;

  return;
}

void SMxSimulator::init(itpp::bvec _inputBits) {
  inputBits = _inputBits;

  Nt = itpp::length(inputBits);

  ofs << "SNR[dB],BER" << std::endl;

  m = log2(nC) * nTx;

  if (Nt % m != 0) {
    inputBits = itpp::concat(inputBits, itpp::zeros_b(Nt % m));
    Nt = itpp::length(inputBits);  // Refresh
  }

  Nu = Nt;

  modulator.set_M(nTx, m);

  Nctx = Nu;
  Nvec = Nctx / m;
  Nbitspvec = m;

  std::cout << m << " " << Nu << " " << Nctx << " " << Nvec << " " << Nbitspvec << std::endl;

  return;
}

itpp::cvec SMSimulator::select_antenna(itpp::bvec bits) {
  itpp::cvec s = itpp::zeros_c(nTx); // アンテナ数
  int Nbits = itpp::length(bits);
  int selected_antenna = itpp::bin2dec(bits(0, nAntenna - 1));

  std::complex<double> symbol = modulator.get_symbols()(itpp::bin2dec(bits(nAntenna, Nbits - 1)));

  if (nAntenna == 0) {
    s(0) = symbol;
  } else {
    s(selected_antenna) = symbol;
  }

  return s;
}

itpp::vec SMSimulator::demodulate_bits(itpp::cvec Y, itpp::cmat H, itpp::cvec S, itpp::Array<itpp::cvec> samples) {
  int i = 0, argmax_P;
  const double coef = pow(itpp::pi, -nRx);

  itpp::vec P(nTx * nSymbols);
  itpp::vec j_q_pair(2);

  for (int j = 0; j < nTx; j++) {
    for (int q = 0; q < nSymbols; q++) {
      if (nTx == 1) {
        P(i++) = -itpp::norm(Y - H * samples(nSymbols * j + q));
      } else {
        P(i++) = coef * exp(-itpp::norm(Y - H * samples(nSymbols * j + q), "fro"));
      }
    }
  }

  argmax_P = itpp::max_index(P);

  j_q_pair(0) = argmax_P / nSymbols;
  j_q_pair(1) = argmax_P % nSymbols;

  return j_q_pair;
}

itpp::Array<itpp::cvec> SMSimulator::get_samples() {
  itpp::cvec symbols = modulator.get_symbols();
  itpp::Array<itpp::cvec> samples(nSymbols * nTx);
  itpp::cvec tmp;

  for (int i = 0; i < nTx; i++) {
    for (int j = 0; j < nSymbols; j++) {
      tmp = itpp::zeros_c(nTx);
      tmp(i) = symbols(j);

      samples(nSymbols * i + j) = tmp;
    }
  }

  return samples;
}

itpp::bvec SMSimulator::simulate(int nsnr) {
  itpp::cvec Y;
  itpp::cmat H;
  itpp::cvec S;
  itpp::cvec e;
  itpp::Array<itpp::cvec> Samples = SMSimulator::get_samples();

  const double sigma2 = itpp::inv_dB(-nsnr);

  itpp::BERC bercu;

  itpp::bvec bitstmp;

  itpp::vec j_q_pair;
  itpp::bvec transmittmp;
  itpp::bvec receivetmp;

  itpp::bvec transmitted(Nctx);
  itpp::bvec received(Nctx);

  for (int k = 0; k < Nvec; k++) {
    H = itpp::randn_c(nRx, nTx);

    bitstmp = inputBits(k * Nbitspvec, (k + 1) * Nbitspvec - 1);

    S = SMSimulator::select_antenna(bitstmp);

    e = sqrt(sigma2) * itpp::randn_c(nRx);

    Y = H * S + e;

    j_q_pair = SMSimulator::demodulate_bits(Y, H, S, Samples);

    transmittmp = bitstmp;
    receivetmp = itpp::concat(itpp::dec2bin(nAntenna, static_cast<int>(j_q_pair(0))),
                              itpp::dec2bin(nSymbin, static_cast<int>(j_q_pair(1)))
                              );

    transmitted.set_subvector(k * Nbitspvec, transmittmp);
    received.set_subvector(k * Nbitspvec, receivetmp);
  }

  bercu.count(transmitted(0, Nu - 1), received(0, Nu - 1));

  std::cout << "-----------------------------------------------------" << std::endl;
  std::cout << "Eb/N0: " << nsnr << " dB. Simulated " << Nctx << " bits." << std::endl;
  std::cout << "BER: " << bercu.get_errorrate() << " (ML)" << std::endl;
  ofs << std::dec << nsnr << "," << std::scientific << bercu.get_errorrate() << std::endl;

  return received;
}

itpp::bvec SMxSimulator::simulate(int nsnr) {
  const int Nmethods = 1;
  const double sigma2 = itpp::inv_dB(-nsnr);

  itpp::Array<itpp::cvec> Y(Nvec);
  itpp::Array<itpp::cmat> H(Nvec + 1);
  itpp::cvec e;

  itpp::BERC bercu;

  itpp::bvec bitstmp;

  itpp::bvec received(Nu);

  for (int k = 0; k < Nvec; k++) {
    H(k) = itpp::randn_c(nRx, nTx);

    bitstmp = inputBits(k * Nbitspvec, (k + 1) * Nbitspvec - 1);

    e = sqrt(sigma2) * itpp::randn_c(nRx);

    itpp::cvec x = modulator.modulate_bits(bitstmp);

    Y(k) = H(k) * x + e;
  }

  // -- demodulate --
  itpp::Array<itpp::QLLRvec> LLRin(Nmethods);
  for (int i = 0; i < Nmethods; i++) {
    LLRin(i) = itpp::zeros_i(Nctx);
  }

  itpp::QLLRvec llr_apr = itpp::zeros_i(Nbitspvec);
  itpp::QLLRvec llr_apost = itpp::zeros_i(Nbitspvec);

  for (int k = 0; k < Nvec; k++) {
    // ML demodulation
    modulator.demodulate_soft_bits(Y(k), H(k), sigma2, llr_apr, llr_apost);
    LLRin(0).set_subvector(k * Nbitspvec, llr_apost);
  }

  // -- decode and count errors --
  for (int i = 0; i < Nmethods; i++) {
    bercu.count(inputBits(0, Nu - 1), LLRin(0)(0, Nu - 1) < 0);
  }

  // -- received bits --
  for (int i = 0; i < Nu; i++) {
    received(i) = LLRin(0)(i) < 0;
  }

  std::cout << "-----------------------------------------------------" << std::endl;
  std::cout << "Eb/N0: " << nsnr << " dB. Simulated " << Nctx << " bits." << std::endl;
  std::cout << "BER: " << bercu.get_errorrate() << " (ML)" << std::endl;
  ofs << std::dec << nsnr << "," << std::scientific << bercu.get_errorrate() << std::endl;

  return received;
}
<|endoftext|>


<commit_before>//$Id: ResultsPostProcessor.h 3456 2013-06-26 02:11:13Z Jamshid $

/*
 * The Software is made available solely for use according to the License Agreement. Any reproduction
 * or redistribution of the Software not in accordance with the License Agreement is expressly prohibited
 * by law, and may result in severe civil and criminal penalties. Violators will be prosecuted to the
 * maximum extent possible.
 *
 * THE SOFTWARE IS WARRANTED, IF AT ALL, ONLY ACCORDING TO THE TERMS OF THE LICENSE AGREEMENT. EXCEPT
 * AS WARRANTED IN THE LICENSE AGREEMENT, SRCH2 INC. HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS WITH
 * REGARD TO THE SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES AND CONDITIONS OF MERCHANTABILITY, FITNESS
 * FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT.  IN NO EVENT SHALL SRCH2 INC. BE LIABLE FOR ANY
 * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA
 * OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER ACTION, ARISING OUT OF OR IN CONNECTION
 * WITH THE USE OR PERFORMANCE OF SOFTWARE.

 * Copyright © 2010 SRCH2 Inc. All rights reserved
 */

#include "FacetedSearchFilterInternal.h"
#include <vector>
#include <map>
#include <string>
#include <algorithm>
#include "util/Assert.h"
#include "instantsearch/TypedValue.h"
#include "util/DateAndTimeHandler.h"

using namespace std;
namespace srch2 {
namespace instantsearch {

float FacetResultsContainer::initAggregation(FacetAggregationType  aggregationType){
	switch (aggregationType) {
		case FacetAggregationTypeCount:
			return 0;
		default:
			ASSERT(false);
			return 0;
	}
}
float FacetResultsContainer::doAggregation(float bucketValue, FacetAggregationType  aggregationType){
	switch (aggregationType) {
		case FacetAggregationTypeCount:
			return bucketValue + 1;
		default:
			ASSERT(false);
			return 0;
	}
}

void CategoricalFacetResultsContainer::initialize(FacetHelper * facetHelper , FacetAggregationType  aggregationType){
	// No need to do anything in this function.
}
void CategoricalFacetResultsContainer::addResultToBucket(const unsigned bucketId, const std::string & bucketName, FacetAggregationType aggregationType){

	std::map<unsigned, std::pair<std::string, float> >::iterator bucketIter = bucketsInfo.find(bucketId);
	if( bucketIter == bucketsInfo.end()){ // A new bucket
		std::pair<std::string, float> newBucket = std::make_pair(bucketName , initAggregation(aggregationType));
		newBucket.second = doAggregation(newBucket.second , aggregationType);
		bucketsInfo[bucketId] = newBucket;
	}else{
		bucketIter->second.second = doAggregation(bucketIter->second.second , aggregationType);
	}

}
void CategoricalFacetResultsContainer::getNamesAndValues(std::vector<std::pair< std::string, float > > & results){
	for(std::map<unsigned, std::pair<std::string, float> >::iterator bucketPtr = bucketsInfo.begin() ;
			bucketPtr != bucketsInfo.end() ; ++bucketPtr){
		results.push_back(std::make_pair(bucketPtr->second.first , bucketPtr->second.second));
	}
}

/*
 * Since we want to include all intervals of a range facet (even if they don't have any records in them)
 * we have to first initialize all intervals to a value (e.g. count of 0). To do this we use facetHelper to give us
 * a list of IDs which it will generate later with their name. For example:
 * <0,"-inf">,<1,"10">,<2,"20">,<3,"30">
 */
void RangeFacetResultsContainer::initialize(FacetHelper * facetHelper , FacetAggregationType  aggregationType){
	std::vector<std::pair<unsigned, std::string> > idsAndNames;
	facetHelper->generateListOfIdsAndNames(&idsAndNames);
	int index = 0;
	for(std::vector<std::pair<unsigned, std::string> >::iterator idAndName = idsAndNames.begin();
			idAndName != idsAndNames.end() ; ++idAndName){
		ASSERT(index == idAndName->first); // bucket Id must be the index of buckets in this vector
		bucketsInfo.push_back(std::make_pair(idAndName->second , initAggregation(aggregationType) ));
		//
		index++;
	}
}
void RangeFacetResultsContainer::addResultToBucket(const unsigned bucketId, const std::string & bucketName, FacetAggregationType aggregationType){
	bucketsInfo.at(bucketId).second = doAggregation(bucketsInfo.at(bucketId).second , aggregationType);
}
void RangeFacetResultsContainer::getNamesAndValues(std::vector<std::pair< std::string, float > > & results){
	results.insert(results.begin(), bucketsInfo.begin() , bucketsInfo.end());
}

std::pair<unsigned , std::string> CategoricalFacetHelper::generateIDAndName(const TypedValue & attributeValue){
	std::string attributeValueLowerCase = attributeValue.toString();
	std::transform(attributeValueLowerCase.begin(), attributeValueLowerCase.end(), attributeValueLowerCase.begin(), ::tolower);

	if(categoryValueToBucketIdMap.find(attributeValueLowerCase) == categoryValueToBucketIdMap.end()){
		categoryValueToBucketIdMap[attributeValueLowerCase] = categoryValueToBucketIdMap.size();
	}
	return std::make_pair(categoryValueToBucketIdMap[attributeValueLowerCase] , attributeValueLowerCase);
}
void CategoricalFacetHelper::generateListOfIdsAndNames(std::vector<std::pair<unsigned, std::string> > * idsAndNames){
	// This function should not be called.
	ASSERT(false);
}
void CategoricalFacetHelper::initialize(const std::string * facetInfoForInitialization, const Schema * schema){
	// No need to do anything here
}

/*
 * This function finds the interval in which attributeValue is placed. ID and name will be returned. since all the names are returned once
 * in generateListOfIdsAndNames, all names are returned as "" to save copying string values.
 */
std::pair<unsigned , std::string> RangeFacetHelper::generateIDAndName(const TypedValue & attributeValue){
	if(attributeValue >= end){
		return std::make_pair(numberOfBuckets-1 , "");
	}
	unsigned bucketId = attributeValue.findIndexOfContainingInterval(start, end, gap);
    if(bucketId == -1){ // Something has gone wrong and Score class has been unable to calculate the index.
    	ASSERT(false);
        return std::make_pair(0 , "");;
    }
    if(bucketId >= numberOfBuckets){
    	bucketId =  numberOfBuckets - 1;
    }
    return std::make_pair(bucketId , "");
}
/*
 * Example :
 * If we have two attributes : price,model (facet type : range, categorical)
 * and start,end and gap for price are 1,100 and 10. Then this function
 * produces an empty vector for model (because it's categorical) and a vector with the following
 * values for price:
 * -large_value, 1, 11, 21, 31, 41, ..., 91, 101
 */
void RangeFacetHelper::generateListOfIdsAndNames(std::vector<std::pair<unsigned, std::string> > * idsAndNames){

	if(generateListOfIdsAndNamesFlag == true || idsAndNames == NULL){
		ASSERT(false);
		return;
	}
	TypedValue lowerBoundToAdd = start;
    std::vector<TypedValue> lowerBounds;
	// Example : start : 1, gap : 10 , end : 100
	// first -large_value is added as the first category
	// then 1, 11, 21, ...and 91 are added in the loop.
	// and 101 is added after loop.
	lowerBounds.push_back(lowerBoundToAdd.minimumValue()); // to collect data smaller than start
	while (lowerBoundToAdd < end) {
		lowerBounds.push_back(lowerBoundToAdd); // data of normal categories
		lowerBoundToAdd = lowerBoundToAdd + gap;
	}
	lowerBounds.push_back(end); // to collect data greater than end
	// now set the number of buckets
	numberOfBuckets = lowerBounds.size();
	// now fill the output
	for(int lowerBoundsIndex = 0; lowerBoundsIndex < lowerBounds.size() ; lowerBoundsIndex++){
		string bucketName = lowerBounds.at(lowerBoundsIndex).toString();
		if(attributeType == ATTRIBUTE_TYPE_TIME){
			long timeValue = lowerBounds.at(lowerBoundsIndex).getTimeTypedValue();
			bucketName = DateAndTimeHandler::convertSecondsFromEpochToDateTimeString(&timeValue);
		}
		idsAndNames->push_back(std::make_pair(lowerBoundsIndex , bucketName));
	}
	// And set the flag to make sure this function is called only once.
	generateListOfIdsAndNamesFlag = false;

}

/*
 * Info must contain :
 * facetInfoForInitialization[0] <= start
 * facetInfoForInitialization[1] <= end
 * facetInfoForInitialization[2] <= gap
 * facetInfoForInitialization[3] <= fieldName
 */
void RangeFacetHelper::initialize(const std::string * facetInfoForInitialization , const Schema * schema){
	std::string startString = facetInfoForInitialization[0];
	std::string endString = facetInfoForInitialization[1];
	std::string gapString = facetInfoForInitialization[2];
	std::string fieldName = facetInfoForInitialization[3];

    attributeType = schema->getTypeOfNonSearchableAttribute(
            schema->getNonSearchableAttributeId(fieldName));
    start.setTypedValue(attributeType, startString);

    end.setTypedValue(attributeType, endString);

    if(attributeType == ATTRIBUTE_TYPE_TIME){
    	// For time attributes gap should not be of the same type, it should be
    	// of type TimeDuration.
    	if(start > end){ // start should not be greater than end
    		start = end;
    		gap.setTypedValue(ATTRIBUTE_TYPE_DURATION, "00:00:00");
    	}else{
    		gap.setTypedValue(ATTRIBUTE_TYPE_DURATION, gapString);
    	}
    }else{
    	if(start > end){ // start should not be greater than end
    		start = end;
    		gap.setTypedValue(attributeType , "0");
    	}else{
    		gap.setTypedValue(attributeType, gapString);
    	}
    }

}

void FacetedSearchFilterInternal::doFilter(IndexSearcher *indexSearcher,
        const Query * query, QueryResults * input, QueryResults * output){
    IndexSearcherInternal * indexSearcherInternal =
            dynamic_cast<IndexSearcherInternal *>(indexSearcher);
    Schema * schema = indexSearcherInternal->getSchema();
    ForwardIndex * forwardIndex = indexSearcherInternal->getForwardIndex();

    // first prepare internal structures based on the input
    preFilter(indexSearcher);

    // also copy all input results to output to save previous filter works
    output->copyForPostProcessing(input);

    // translate list of attribute names to list of attribute IDs
    std::vector<unsigned> attributeIds;
    for(std::vector<std::string>::iterator facetField = fields.begin();
            facetField != fields.end() ; ++facetField){
        attributeIds.push_back(schema->getNonSearchableAttributeId(*facetField));
    }

    // move on the results once and do all facet calculations.
    for (std::vector<QueryResult *>::iterator resultIter =
            output->impl->sortedFinalResults.begin();
            resultIter != output->impl->sortedFinalResults.end();
            ++resultIter) {
        QueryResult * queryResult = *resultIter;
        // extract all facet related nonsearchable attribute values from this record
        // by accessing the forward index only once.
        bool isValid = false;
        const ForwardList * list = forwardIndex->getForwardList(
                queryResult->internalRecordId, isValid);
        ASSERT(isValid);
        const Byte * nonSearchableAttributesData =
                list->getNonSearchableAttributeContainerData();
        // this vector is parallel to attributeIds vector
        std::vector<TypedValue> attributeDataValues;
        VariableLengthAttributeContainer::getBatchOfAttributes(attributeIds, schema,nonSearchableAttributesData, &attributeDataValues);

        // now iterate on attributes and incrementally update the facet results
        for(std::vector<std::string>::iterator facetField = fields.begin();
                facetField != fields.end() ; ++facetField){
        	TypedValue & attributeValue = attributeDataValues.at(
                                   std::distance(fields.begin() , facetField));
            // choose the type of aggregation for this attribute
            // increments the correct facet by one
            doProcessOneResult(attributeValue , std::distance(fields.begin() , facetField));
        }
    }

    // now copy all results to output
	for(std::vector<std::pair< FacetType , FacetResultsContainer * > >::iterator facetResultsPtr = facetResults.begin();
			facetResultsPtr != facetResults.end(); ++facetResultsPtr){
		std::vector<std::pair< std::string, float > > results;
		facetResultsPtr->second->getNamesAndValues(results);
		output->impl->facetResults[fields.at(std::distance(facetResults.begin() , facetResultsPtr))] = std::make_pair(facetResultsPtr->first , results);
	}

}

void FacetedSearchFilterInternal::preFilter(IndexSearcher *indexSearcher){

    IndexSearcherInternal * indexSearcherInternal =
            dynamic_cast<IndexSearcherInternal *>(indexSearcher);
    Schema * schema = indexSearcherInternal->getSchema();

	for(std::vector<std::string>::iterator facetField = fields.begin();
            facetField != fields.end() ; ++facetField){
		FacetType facetType = facetTypes.at(std::distance(fields.begin() , facetField));
		FacetHelper * facetHelper = NULL;
		FacetResultsContainer * facetResultsContainer = NULL;
		switch (facetType) {
			case FacetTypeCategorical:
				facetHelper = new CategoricalFacetHelper();
				facetResultsContainer = new CategoricalFacetResultsContainer();
				break;
			case FacetTypeRange:
				facetHelper = new RangeFacetHelper();
				facetResultsContainer = new RangeFacetResultsContainer();
				break;
			default:
				ASSERT(false);
				break;
		}
		std::string info[4];
		info[0] = rangeStarts.at(std::distance(fields.begin() , facetField));
		info[1] = rangeEnds.at(std::distance(fields.begin() , facetField));
		info[2] = rangeGaps.at(std::distance(fields.begin() , facetField));
		info[3] = *facetField;
		facetHelper->initialize(info , schema);
		facetResultsContainer->initialize(facetHelper , FacetAggregationTypeCount );
		this->facetResults.push_back(std::make_pair(facetType , facetResultsContainer));
		this->facetHelpers.push_back(facetHelper);

	}
}
void FacetedSearchFilterInternal::doProcessOneResult(const TypedValue & attributeValue, const unsigned facetFieldIndex){
	std::pair<unsigned , std::string> idandName = this->facetHelpers.at(facetFieldIndex)->generateIDAndName(attributeValue);
	this->facetResults.at(facetFieldIndex).second->addResultToBucket(idandName.first , idandName.second , FacetAggregationTypeCount);
}

void FacetedSearchFilterInternal::initialize(std::vector<FacetType> & facetTypes,
        std::vector<std::string> & fields, std::vector<std::string> & rangeStarts,
        std::vector<std::string> & rangeEnds,
        std::vector<std::string> & rangeGaps){
    this->fields = fields;
    this->facetTypes = facetTypes;
    this->rangeStarts = rangeStarts;
    this->rangeEnds = rangeEnds;
    this->rangeGaps = rangeGaps;
}

}
}
<commit_msg>Code review comments are applied.<commit_after>//$Id: ResultsPostProcessor.h 3456 2013-06-26 02:11:13Z Jamshid $

/*
 * The Software is made available solely for use according to the License Agreement. Any reproduction
 * or redistribution of the Software not in accordance with the License Agreement is expressly prohibited
 * by law, and may result in severe civil and criminal penalties. Violators will be prosecuted to the
 * maximum extent possible.
 *
 * THE SOFTWARE IS WARRANTED, IF AT ALL, ONLY ACCORDING TO THE TERMS OF THE LICENSE AGREEMENT. EXCEPT
 * AS WARRANTED IN THE LICENSE AGREEMENT, SRCH2 INC. HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS WITH
 * REGARD TO THE SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES AND CONDITIONS OF MERCHANTABILITY, FITNESS
 * FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT.  IN NO EVENT SHALL SRCH2 INC. BE LIABLE FOR ANY
 * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA
 * OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER ACTION, ARISING OUT OF OR IN CONNECTION
 * WITH THE USE OR PERFORMANCE OF SOFTWARE.

 * Copyright © 2010 SRCH2 Inc. All rights reserved
 */

#include "FacetedSearchFilterInternal.h"
#include <vector>
#include <map>
#include <string>
#include <algorithm>
#include "util/Assert.h"
#include "instantsearch/TypedValue.h"
#include "util/DateAndTimeHandler.h"

using namespace std;
namespace srch2 {
namespace instantsearch {

float FacetResultsContainer::initAggregation(FacetAggregationType  aggregationType){
	switch (aggregationType) {
		case FacetAggregationTypeCount:
			return 0;
		default:
			ASSERT(false);
			return 0;
	}
}
float FacetResultsContainer::doAggregation(float bucketValue, FacetAggregationType  aggregationType){
	switch (aggregationType) {
		case FacetAggregationTypeCount:
			return bucketValue + 1;
		default:
			ASSERT(false);
			return 0;
	}
}

void CategoricalFacetResultsContainer::initialize(FacetHelper * facetHelper , FacetAggregationType  aggregationType){
	// No need to do anything in this function.
}
void CategoricalFacetResultsContainer::addResultToBucket(const unsigned bucketId, const std::string & bucketName, FacetAggregationType aggregationType){

	std::map<unsigned, std::pair<std::string, float> >::iterator bucketIter = bucketsInfo.find(bucketId);
	if( bucketIter == bucketsInfo.end()){ // A new bucket
		std::pair<std::string, float> newBucket = std::make_pair(bucketName , initAggregation(aggregationType));
		newBucket.second = doAggregation(newBucket.second , aggregationType);
		bucketsInfo[bucketId] = newBucket;
	}else{
		bucketIter->second.second = doAggregation(bucketIter->second.second , aggregationType);
	}

}
void CategoricalFacetResultsContainer::getNamesAndValues(std::vector<std::pair< std::string, float > > & results){
	for(std::map<unsigned, std::pair<std::string, float> >::iterator bucketPtr = bucketsInfo.begin() ;
			bucketPtr != bucketsInfo.end() ; ++bucketPtr){
		results.push_back(std::make_pair(bucketPtr->second.first , bucketPtr->second.second));
	}
}

/*
 * Since we want to include all intervals of a range facet (even if they don't have any records in them)
 * we have to first initialize all intervals to a value (e.g. count of 0). To do this we use facetHelper to give us
 * a list of IDs which it will generate later with their name. For example:
 * <0,"-inf">,<1,"10">,<2,"20">,<3,"30">
 */
void RangeFacetResultsContainer::initialize(FacetHelper * facetHelper , FacetAggregationType  aggregationType){
	std::vector<std::pair<unsigned, std::string> > idsAndNames;
	facetHelper->generateListOfIdsAndNames(&idsAndNames);
	int index = 0;
	for(std::vector<std::pair<unsigned, std::string> >::iterator idAndName = idsAndNames.begin();
			idAndName != idsAndNames.end() ; ++idAndName){
		ASSERT(index == idAndName->first); // bucket Id must be the index of buckets in this vector
		bucketsInfo.push_back(std::make_pair(idAndName->second , initAggregation(aggregationType) ));
		//
		index++;
	}
}
void RangeFacetResultsContainer::addResultToBucket(const unsigned bucketId, const std::string & bucketName, FacetAggregationType aggregationType){
	bucketsInfo.at(bucketId).second = doAggregation(bucketsInfo.at(bucketId).second , aggregationType);
}
void RangeFacetResultsContainer::getNamesAndValues(std::vector<std::pair< std::string, float > > & results){
	results.insert(results.begin(), bucketsInfo.begin() , bucketsInfo.end());
}

std::pair<unsigned , std::string> CategoricalFacetHelper::generateIDAndName(const TypedValue & attributeValue){
	std::string attributeValueLowerCase = attributeValue.toString();
	std::transform(attributeValueLowerCase.begin(), attributeValueLowerCase.end(), attributeValueLowerCase.begin(), ::tolower);

	if(categoryValueToBucketIdMap.find(attributeValueLowerCase) == categoryValueToBucketIdMap.end()){
		categoryValueToBucketIdMap[attributeValueLowerCase] = categoryValueToBucketIdMap.size();
	}
	return std::make_pair(categoryValueToBucketIdMap[attributeValueLowerCase] , attributeValueLowerCase);
}
void CategoricalFacetHelper::generateListOfIdsAndNames(std::vector<std::pair<unsigned, std::string> > * idsAndNames){
	// This function should not be called.
	ASSERT(false);
}
void CategoricalFacetHelper::initialize(const std::string * facetInfoForInitialization, const Schema * schema){
	// No need to do anything here
}

/*
 * This function finds the interval in which attributeValue is placed. ID and name will be returned. since all the names are returned once
 * in generateListOfIdsAndNames, all names are returned as "" to save copying string values.
 */
std::pair<unsigned , std::string> RangeFacetHelper::generateIDAndName(const TypedValue & attributeValue){
	if(attributeValue >= end){
		return std::make_pair(numberOfBuckets-1 , "");
	}
	unsigned bucketId = attributeValue.findIndexOfContainingInterval(start, end, gap);
    if(bucketId == -1){ // Something has gone wrong and Score class has been unable to calculate the index.
    	ASSERT(false);
        return std::make_pair(0 , "");;
    }
    if(bucketId >= numberOfBuckets){
    	bucketId =  numberOfBuckets - 1;
    }
    return std::make_pair(bucketId , "");
}
/*
 * Example :
 * If we have two attributes : price,model (facet type : range, categorical)
 * and start,end and gap for price are 1,100 and 10. Then this function
 * produces an empty vector for model (because it's categorical) and a vector with the following
 * values for price:
 * -large_value, 1, 11, 21, 31, 41, ..., 91, 101
 */
void RangeFacetHelper::generateListOfIdsAndNames(std::vector<std::pair<unsigned, std::string> > * idsAndNames){

	if(generateListOfIdsAndNamesFlag == true || idsAndNames == NULL){
		ASSERT(false);
		return;
	}
	TypedValue lowerBoundToAdd = start;
    std::vector<TypedValue> lowerBounds;
	// Example : start : 1, gap : 10 , end : 100
	// first -large_value is added as the first category
	// then 1, 11, 21, ...and 91 are added in the loop.
	// and 101 is added after loop.
	lowerBounds.push_back(lowerBoundToAdd.minimumValue()); // to collect data smaller than start
	while (lowerBoundToAdd < end) {
		lowerBounds.push_back(lowerBoundToAdd); // data of normal categories
		lowerBoundToAdd = lowerBoundToAdd + gap;
	}
	lowerBounds.push_back(end); // to collect data greater than end
	// now set the number of buckets
	numberOfBuckets = lowerBounds.size();
	// now fill the output
	for(int lowerBoundsIndex = 0; lowerBoundsIndex < lowerBounds.size() ; lowerBoundsIndex++){
		string bucketName = lowerBounds.at(lowerBoundsIndex).toString();
		/*
		 * If the type of this facet attribute is time, we should translate the number of
		 * seconds from Jan 1st 1970 (aka "epoch") to a human readable representation of time.
		 * For example, if this value is 1381271294, the name of this bucket is 10/8/2013 3:28:14.
		 */
		if(attributeType == ATTRIBUTE_TYPE_TIME){
			long timeValue = lowerBounds.at(lowerBoundsIndex).getTimeTypedValue();
			bucketName = DateAndTimeHandler::convertSecondsFromEpochToDateTimeString(&timeValue);
		}
		idsAndNames->push_back(std::make_pair(lowerBoundsIndex , bucketName));
	}
	// And set the flag to make sure this function is called only once.
	generateListOfIdsAndNamesFlag = false;

}

/*
 * Info must contain :
 * facetInfoForInitialization[0] <= start
 * facetInfoForInitialization[1] <= end
 * facetInfoForInitialization[2] <= gap
 * facetInfoForInitialization[3] <= fieldName
 */
void RangeFacetHelper::initialize(const std::string * facetInfoForInitialization , const Schema * schema){
	std::string startString = facetInfoForInitialization[0];
	std::string endString = facetInfoForInitialization[1];
	std::string gapString = facetInfoForInitialization[2];
	std::string fieldName = facetInfoForInitialization[3];

    attributeType = schema->getTypeOfNonSearchableAttribute(
            schema->getNonSearchableAttributeId(fieldName));
    start.setTypedValue(attributeType, startString);

    end.setTypedValue(attributeType, endString);

    if(attributeType == ATTRIBUTE_TYPE_TIME){
    	// For time attributes gap should not be of the same type, it should be
    	// of type TimeDuration.
    	if(start > end){ // start should not be greater than end
    		start = end;
    		gap.setTypedValue(ATTRIBUTE_TYPE_DURATION, "00:00:00");
    	}else{
    		gap.setTypedValue(ATTRIBUTE_TYPE_DURATION, gapString);
    	}
    }else{
    	if(start > end){ // start should not be greater than end
    		start = end;
    		gap.setTypedValue(attributeType , "0");
    	}else{
    		gap.setTypedValue(attributeType, gapString);
    	}
    }

}

void FacetedSearchFilterInternal::doFilter(IndexSearcher *indexSearcher,
        const Query * query, QueryResults * input, QueryResults * output){
    IndexSearcherInternal * indexSearcherInternal =
            dynamic_cast<IndexSearcherInternal *>(indexSearcher);
    Schema * schema = indexSearcherInternal->getSchema();
    ForwardIndex * forwardIndex = indexSearcherInternal->getForwardIndex();

    // first prepare internal structures based on the input
    preFilter(indexSearcher);

    // also copy all input results to output to save previous filter works
    output->copyForPostProcessing(input);

    // translate list of attribute names to list of attribute IDs
    std::vector<unsigned> attributeIds;
    for(std::vector<std::string>::iterator facetField = fields.begin();
            facetField != fields.end() ; ++facetField){
        attributeIds.push_back(schema->getNonSearchableAttributeId(*facetField));
    }

    // move on the results once and do all facet calculations.
    for (std::vector<QueryResult *>::iterator resultIter =
            output->impl->sortedFinalResults.begin();
            resultIter != output->impl->sortedFinalResults.end();
            ++resultIter) {
        QueryResult * queryResult = *resultIter;
        // extract all facet related nonsearchable attribute values from this record
        // by accessing the forward index only once.
        bool isValid = false;
        const ForwardList * list = forwardIndex->getForwardList(
                queryResult->internalRecordId, isValid);
        ASSERT(isValid);
        const Byte * nonSearchableAttributesData =
                list->getNonSearchableAttributeContainerData();
        // this vector is parallel to attributeIds vector
        std::vector<TypedValue> attributeDataValues;
        VariableLengthAttributeContainer::getBatchOfAttributes(attributeIds, schema,nonSearchableAttributesData, &attributeDataValues);

        // now iterate on attributes and incrementally update the facet results
        for(std::vector<std::string>::iterator facetField = fields.begin();
                facetField != fields.end() ; ++facetField){
        	TypedValue & attributeValue = attributeDataValues.at(
                                   std::distance(fields.begin() , facetField));
            // choose the type of aggregation for this attribute
            // increments the correct facet by one
            doProcessOneResult(attributeValue , std::distance(fields.begin() , facetField));
        }
    }

    // now copy all results to output
	for(std::vector<std::pair< FacetType , FacetResultsContainer * > >::iterator facetResultsPtr = facetResults.begin();
			facetResultsPtr != facetResults.end(); ++facetResultsPtr){
		std::vector<std::pair< std::string, float > > results;
		facetResultsPtr->second->getNamesAndValues(results);
		output->impl->facetResults[fields.at(std::distance(facetResults.begin() , facetResultsPtr))] = std::make_pair(facetResultsPtr->first , results);
	}

}

void FacetedSearchFilterInternal::preFilter(IndexSearcher *indexSearcher){

    IndexSearcherInternal * indexSearcherInternal =
            dynamic_cast<IndexSearcherInternal *>(indexSearcher);
    Schema * schema = indexSearcherInternal->getSchema();

	for(std::vector<std::string>::iterator facetField = fields.begin();
            facetField != fields.end() ; ++facetField){
		FacetType facetType = facetTypes.at(std::distance(fields.begin() , facetField));
		FacetHelper * facetHelper = NULL;
		FacetResultsContainer * facetResultsContainer = NULL;
		switch (facetType) {
			case FacetTypeCategorical:
				facetHelper = new CategoricalFacetHelper();
				facetResultsContainer = new CategoricalFacetResultsContainer();
				break;
			case FacetTypeRange:
				facetHelper = new RangeFacetHelper();
				facetResultsContainer = new RangeFacetResultsContainer();
				break;
			default:
				ASSERT(false);
				break;
		}
		std::string info[4];
		info[0] = rangeStarts.at(std::distance(fields.begin() , facetField));
		info[1] = rangeEnds.at(std::distance(fields.begin() , facetField));
		info[2] = rangeGaps.at(std::distance(fields.begin() , facetField));
		info[3] = *facetField;
		facetHelper->initialize(info , schema);
		facetResultsContainer->initialize(facetHelper , FacetAggregationTypeCount );
		this->facetResults.push_back(std::make_pair(facetType , facetResultsContainer));
		this->facetHelpers.push_back(facetHelper);

	}
}
void FacetedSearchFilterInternal::doProcessOneResult(const TypedValue & attributeValue, const unsigned facetFieldIndex){
	std::pair<unsigned , std::string> idandName = this->facetHelpers.at(facetFieldIndex)->generateIDAndName(attributeValue);
	this->facetResults.at(facetFieldIndex).second->addResultToBucket(idandName.first , idandName.second , FacetAggregationTypeCount);
}

void FacetedSearchFilterInternal::initialize(std::vector<FacetType> & facetTypes,
        std::vector<std::string> & fields, std::vector<std::string> & rangeStarts,
        std::vector<std::string> & rangeEnds,
        std::vector<std::string> & rangeGaps){
    this->fields = fields;
    this->facetTypes = facetTypes;
    this->rangeStarts = rangeStarts;
    this->rangeEnds = rangeEnds;
    this->rangeGaps = rangeGaps;
}

}
}
<|endoftext|>


<commit_before>/*
 *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

/*
 * This file includes unit tests for NetEQ.
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>  // memset

#include <string>
#include <vector>

#include "gtest/gtest.h"

#include "modules/audio_coding/neteq/test/NETEQTEST_CodecClass.h"
#include "modules/audio_coding/neteq/test/NETEQTEST_NetEQClass.h"
#include "modules/audio_coding/neteq/test/NETEQTEST_RTPpacket.h"
#include "typedefs.h"  // NOLINT(build/include)
#include "modules/audio_coding/neteq/interface/webrtc_neteq.h"
#include "modules/audio_coding/neteq/interface/webrtc_neteq_help_macros.h"
#include "testsupport/fileutils.h"

namespace webrtc {

class NetEqDecodingTest : public ::testing::Test {
 protected:
  NetEqDecodingTest();
  virtual void SetUp();
  virtual void TearDown();
  void SelectDecoders(WebRtcNetEQDecoder* used_codec);
  void LoadDecoders();
  void DecodeAndCompare(const std::string &rtp_file,
                        const std::string &ref_file);

  NETEQTEST_NetEQClass* neteq_inst_;
  std::vector<NETEQTEST_Decoder*> dec_;
};

NetEqDecodingTest::NetEqDecodingTest() : neteq_inst_(NULL) {}

void NetEqDecodingTest::SetUp() {
  WebRtcNetEQDecoder usedCodec[kDecoderReservedEnd - 1];

  SelectDecoders(usedCodec);
  neteq_inst_ = new NETEQTEST_NetEQClass(usedCodec, dec_.size(), 8000,
                                         kTCPLargeJitter);
  ASSERT_TRUE(neteq_inst_);
  LoadDecoders();
}

void NetEqDecodingTest::TearDown() {
  if (neteq_inst_)
    delete neteq_inst_;
  for (size_t i = 0; i < dec_.size(); ++i) {
    if (dec_[i])
      delete dec_[i];
  }
}

void NetEqDecodingTest::SelectDecoders(WebRtcNetEQDecoder* used_codec) {
  *used_codec++ = kDecoderPCMu;
  dec_.push_back(new decoder_PCMU(0));
  *used_codec++ = kDecoderPCMa;
  dec_.push_back(new decoder_PCMA(8));
  *used_codec++ = kDecoderILBC;
  dec_.push_back(new decoder_ILBC(102));
  *used_codec++ = kDecoderISAC;
  dec_.push_back(new decoder_iSAC(103));
  *used_codec++ = kDecoderISACswb;
  dec_.push_back(new decoder_iSACSWB(104));
  *used_codec++ = kDecoderPCM16B;
  dec_.push_back(new decoder_PCM16B_NB(93));
  *used_codec++ = kDecoderPCM16Bwb;
  dec_.push_back(new decoder_PCM16B_WB(94));
  *used_codec++ = kDecoderPCM16Bswb32kHz;
  dec_.push_back(new decoder_PCM16B_SWB32(95));
  *used_codec++ = kDecoderCNG;
  dec_.push_back(new decoder_CNG(13));
}

void NetEqDecodingTest::LoadDecoders() {
  for (size_t i = 0; i < dec_.size(); ++i) {
    ASSERT_EQ(0, dec_[i]->loadToNetEQ(*neteq_inst_));
  }
}

void NetEqDecodingTest::DecodeAndCompare(const std::string &rtp_file,
                                         const std::string &ref_file) {
  NETEQTEST_RTPpacket rtp;
  FILE* rtp_fp = fopen(rtp_file.c_str(), "rb");
  ASSERT_TRUE(rtp_fp != NULL);
  ASSERT_EQ(0, NETEQTEST_RTPpacket::skipFileHeader(rtp_fp));
  ASSERT_GT(rtp.readFromFile(rtp_fp), 0);

  FILE* ref_fp = NULL;
  FILE* out_fp = NULL;
  if (!ref_file.empty()) {
    ref_fp = fopen(ref_file.c_str(), "rb");
    ASSERT_TRUE(ref_fp != NULL);
  } else {
    std::string out_file = webrtc::test::OutputPath() + "neteq_out.pcm";
    out_fp = fopen(out_file.c_str(), "wb");
    ASSERT_TRUE(out_fp != NULL);
  }

  unsigned int sim_clock = 0;
  // NetEQ must be polled for data once every 10 ms. Thus, neither of the
  // constants below can be changed.
  const int kTimeStepMs = 10;
  const int kBlockSize8kHz = kTimeStepMs * 8;
  const int kBlockSize16kHz = kTimeStepMs * 16;
  const int kBlockSize32kHz = kTimeStepMs * 32;
  const int kMaxBlockSize = kBlockSize32kHz;
  while (rtp.dataLen() >= 0) {
    // Check if time to receive.
    while ((sim_clock >= rtp.time()) &&
           (rtp.dataLen() >= 0)) {
      if (rtp.dataLen() > 0) {
        ASSERT_EQ(0, neteq_inst_->recIn(rtp));
      }
      // Get next packet.
      ASSERT_NE(-1, rtp.readFromFile(rtp_fp));
    }

    // RecOut
    WebRtc_Word16 out_data[kMaxBlockSize];
    WebRtc_Word16 out_len = neteq_inst_->recOut(out_data);
    ASSERT_TRUE((out_len == kBlockSize8kHz) ||
                (out_len == kBlockSize16kHz) ||
                (out_len == kBlockSize32kHz));

    if (ref_fp) {
      // Read from ref file.
      WebRtc_Word16 ref_data[kMaxBlockSize];
      if (static_cast<size_t>(out_len) !=
          fread(ref_data, sizeof(WebRtc_Word16), out_len, ref_fp)) {
        break;
      }

      // Compare
      EXPECT_EQ(0, memcmp(out_data, ref_data, sizeof(WebRtc_Word16) * out_len));
    }

    if (out_fp) {
      // Write to output file (mainly for generating new output vectors).
      ASSERT_EQ(static_cast<size_t>(out_len),
                fwrite(out_data, sizeof(WebRtc_Word16), out_len, out_fp));
    }

    // Increase time.
    sim_clock += kTimeStepMs;
  }
  fclose(rtp_fp);
  if (ref_fp) {
    ASSERT_NE(0, feof(ref_fp));  // Make sure that we reached the end.
    fclose(ref_fp);
  }
  if (out_fp) fclose(out_fp);
}

#if defined(WEBRTC_LINUX) && defined(WEBRTC_ARCH_64_BITS)
TEST_F(NetEqDecodingTest, TestBitExactness) {
  const std::string kInputRtpFile = webrtc::test::ProjectRootPath() +
      "test/data/audio_coding/universal.rtp";
  const std::string kInputRefFile = webrtc::test::ProjectRootPath() +
      "test/data/audio_coding/universal_ref.pcm";
  DecodeAndCompare(kInputRtpFile, kInputRefFile);
}
#endif  // defined(WEBRTC_LINUX) && defined(WEBRTC_ARCH_64_BITS)

}  // namespace
<commit_msg>Testing NetEQ network statistics<commit_after>/*
 *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

/*
 * This file includes unit tests for NetEQ.
 */

#include <stdlib.h>
#include <string.h>  // memset

#include <string>
#include <vector>

#include "gtest/gtest.h"

#include "modules/audio_coding/neteq/interface/webrtc_neteq.h"
#include "modules/audio_coding/neteq/interface/webrtc_neteq_help_macros.h"
#include "modules/audio_coding/neteq/test/NETEQTEST_CodecClass.h"
#include "modules/audio_coding/neteq/test/NETEQTEST_NetEQClass.h"
#include "modules/audio_coding/neteq/test/NETEQTEST_RTPpacket.h"
#include "testsupport/fileutils.h"
#include "typedefs.h"  // NOLINT(build/include)

namespace webrtc {

class RefFiles {
 public:
  RefFiles(const std::string& input_file, const std::string& output_file);
  ~RefFiles();
  template<class T> void ProcessReference(const T& test_results);
  template<typename T, size_t n> void ProcessReference(
      const T (&test_results)[n],
      size_t length);
  template<typename T, size_t n> void WriteToFile(
      const T (&test_results)[n],
      size_t length);
  template<typename T, size_t n> void ReadFromFileAndCompare(
      const T (&test_results)[n],
      size_t length);
  void WriteToFile(const WebRtcNetEQ_NetworkStatistics& stats);
  void ReadFromFileAndCompare(const WebRtcNetEQ_NetworkStatistics& stats);
  void WriteToFile(const WebRtcNetEQ_RTCPStat& stats);
  void ReadFromFileAndCompare(const WebRtcNetEQ_RTCPStat& stats);

  FILE* input_fp_;
  FILE* output_fp_;
};

RefFiles::RefFiles(const std::string &input_file,
                   const std::string &output_file)
    : input_fp_(NULL),
      output_fp_(NULL) {
  if (!input_file.empty()) {
    input_fp_ = fopen(input_file.c_str(), "rb");
    EXPECT_TRUE(input_fp_ != NULL);
  }
  if (!output_file.empty()) {
    output_fp_ = fopen(output_file.c_str(), "wb");
    EXPECT_TRUE(output_fp_ != NULL);
  }
}

RefFiles::~RefFiles() {
  if (input_fp_) {
    EXPECT_EQ(EOF, fgetc(input_fp_));  // Make sure that we reached the end.
    fclose(input_fp_);
  }
  if (output_fp_) fclose(output_fp_);
}

template<class T>
void RefFiles::ProcessReference(const T& test_results) {
  WriteToFile(test_results);
  ReadFromFileAndCompare(test_results);
}

template<typename T, size_t n>
void RefFiles::ProcessReference(const T (&test_results)[n], size_t length) {
  WriteToFile(test_results, length);
  ReadFromFileAndCompare(test_results, length);
}

template<typename T, size_t n>
void RefFiles::WriteToFile(const T (&test_results)[n], size_t length) {
  if (output_fp_) {
    ASSERT_EQ(length, fwrite(&test_results, sizeof(T), length, output_fp_));
  }
}

template<typename T, size_t n>
void RefFiles::ReadFromFileAndCompare(const T (&test_results)[n],
                                      size_t length) {
  if (input_fp_) {
    // Read from ref file.
    T* ref = new T[length];
    ASSERT_EQ(length, fread(ref, sizeof(T), length, input_fp_));
    // Compare
    EXPECT_EQ(0, memcmp(&test_results, ref, sizeof(T) * length));
    delete [] ref;
  }
}

void RefFiles::WriteToFile(const WebRtcNetEQ_NetworkStatistics& stats) {
  if (output_fp_) {
    ASSERT_EQ(1u, fwrite(&stats, sizeof(WebRtcNetEQ_NetworkStatistics), 1,
                         output_fp_));
  }
}

void RefFiles::ReadFromFileAndCompare(
    const WebRtcNetEQ_NetworkStatistics& stats) {
  if (input_fp_) {
    // Read from ref file.
    size_t stat_size = sizeof(WebRtcNetEQ_NetworkStatistics);
    WebRtcNetEQ_NetworkStatistics ref_stats;
    ASSERT_EQ(1u, fread(&ref_stats, stat_size, 1, input_fp_));
    // Compare
    EXPECT_EQ(0, memcmp(&stats, &ref_stats, stat_size));
  }
}

void RefFiles::WriteToFile(const WebRtcNetEQ_RTCPStat& stats) {
  if (output_fp_) {
    ASSERT_EQ(1u, fwrite(&(stats.fraction_lost), sizeof(stats.fraction_lost), 1,
                         output_fp_));
    ASSERT_EQ(1u, fwrite(&(stats.cum_lost), sizeof(stats.cum_lost), 1,
                         output_fp_));
    ASSERT_EQ(1u, fwrite(&(stats.ext_max), sizeof(stats.ext_max), 1,
                         output_fp_));
    ASSERT_EQ(1u, fwrite(&(stats.jitter), sizeof(stats.jitter), 1,
                         output_fp_));
  }
}

void RefFiles::ReadFromFileAndCompare(
    const WebRtcNetEQ_RTCPStat& stats) {
  if (input_fp_) {
    // Read from ref file.
    WebRtcNetEQ_RTCPStat ref_stats;
    ASSERT_EQ(1u, fread(&(ref_stats.fraction_lost),
                        sizeof(ref_stats.fraction_lost), 1, input_fp_));
    ASSERT_EQ(1u, fread(&(ref_stats.cum_lost), sizeof(ref_stats.cum_lost), 1,
                        input_fp_));
    ASSERT_EQ(1u, fread(&(ref_stats.ext_max), sizeof(ref_stats.ext_max), 1,
                        input_fp_));
    ASSERT_EQ(1u, fread(&(ref_stats.jitter), sizeof(ref_stats.jitter), 1,
                        input_fp_));
    // Compare
    EXPECT_EQ(ref_stats.fraction_lost, stats.fraction_lost);
    EXPECT_EQ(ref_stats.cum_lost, stats.cum_lost);
    EXPECT_EQ(ref_stats.ext_max, stats.ext_max);
    EXPECT_EQ(ref_stats.jitter, stats.jitter);
  }
}

class NetEqDecodingTest : public ::testing::Test {
 protected:
  // NetEQ must be polled for data once every 10 ms. Thus, neither of the
  // constants below can be changed.
  static const int kTimeStepMs = 10;
  static const int kBlockSize8kHz = kTimeStepMs * 8;
  static const int kBlockSize16kHz = kTimeStepMs * 16;
  static const int kBlockSize32kHz = kTimeStepMs * 32;
  static const int kMaxBlockSize = kBlockSize32kHz;

  NetEqDecodingTest();
  virtual void SetUp();
  virtual void TearDown();
  void SelectDecoders(WebRtcNetEQDecoder* used_codec);
  void LoadDecoders();
  void OpenInputFile(const std::string &rtp_file);
  void Process(NETEQTEST_RTPpacket* rtp_ptr, int16_t* out_len);
  void DecodeAndCompare(const std::string &rtp_file,
                        const std::string &ref_file);
  void DecodeAndCheckStats(const std::string &rtp_file,
                           const std::string &stat_ref_file,
                           const std::string &rtcp_ref_file);

  NETEQTEST_NetEQClass* neteq_inst_;
  std::vector<NETEQTEST_Decoder*> dec_;
  FILE* rtp_fp_;
  unsigned int sim_clock_;
  int16_t out_data_[kMaxBlockSize];
};

NetEqDecodingTest::NetEqDecodingTest()
    : neteq_inst_(NULL),
      rtp_fp_(NULL),
      sim_clock_(0) {
  memset(out_data_, 0, sizeof(out_data_));
}

void NetEqDecodingTest::SetUp() {
  WebRtcNetEQDecoder usedCodec[kDecoderReservedEnd - 1];

  SelectDecoders(usedCodec);
  neteq_inst_ = new NETEQTEST_NetEQClass(usedCodec, dec_.size(), 8000,
                                         kTCPLargeJitter);
  ASSERT_TRUE(neteq_inst_);
  LoadDecoders();
}

void NetEqDecodingTest::TearDown() {
  if (neteq_inst_)
    delete neteq_inst_;
  for (size_t i = 0; i < dec_.size(); ++i) {
    if (dec_[i])
      delete dec_[i];
  }
  if (rtp_fp_)
    fclose(rtp_fp_);
}

void NetEqDecodingTest::SelectDecoders(WebRtcNetEQDecoder* used_codec) {
  *used_codec++ = kDecoderPCMu;
  dec_.push_back(new decoder_PCMU(0));
  *used_codec++ = kDecoderPCMa;
  dec_.push_back(new decoder_PCMA(8));
  *used_codec++ = kDecoderILBC;
  dec_.push_back(new decoder_ILBC(102));
  *used_codec++ = kDecoderISAC;
  dec_.push_back(new decoder_iSAC(103));
  *used_codec++ = kDecoderISACswb;
  dec_.push_back(new decoder_iSACSWB(104));
  *used_codec++ = kDecoderPCM16B;
  dec_.push_back(new decoder_PCM16B_NB(93));
  *used_codec++ = kDecoderPCM16Bwb;
  dec_.push_back(new decoder_PCM16B_WB(94));
  *used_codec++ = kDecoderPCM16Bswb32kHz;
  dec_.push_back(new decoder_PCM16B_SWB32(95));
  *used_codec++ = kDecoderCNG;
  dec_.push_back(new decoder_CNG(13));
}

void NetEqDecodingTest::LoadDecoders() {
  for (size_t i = 0; i < dec_.size(); ++i) {
    ASSERT_EQ(0, dec_[i]->loadToNetEQ(*neteq_inst_));
  }
}

void NetEqDecodingTest::OpenInputFile(const std::string &rtp_file) {
  rtp_fp_ = fopen(rtp_file.c_str(), "rb");
  ASSERT_TRUE(rtp_fp_ != NULL);
  ASSERT_EQ(0, NETEQTEST_RTPpacket::skipFileHeader(rtp_fp_));
}

void NetEqDecodingTest::Process(NETEQTEST_RTPpacket* rtp, int16_t* out_len) {
  // Check if time to receive.
  while ((sim_clock_ >= rtp->time()) &&
         (rtp->dataLen() >= 0)) {
    if (rtp->dataLen() > 0) {
      ASSERT_EQ(0, neteq_inst_->recIn(*rtp));
    }
    // Get next packet.
    ASSERT_NE(-1, rtp->readFromFile(rtp_fp_));
  }

  // RecOut
  *out_len = neteq_inst_->recOut(out_data_);
  ASSERT_TRUE((*out_len == kBlockSize8kHz) ||
              (*out_len == kBlockSize16kHz) ||
              (*out_len == kBlockSize32kHz));

  // Increase time.
  sim_clock_ += kTimeStepMs;
}

void NetEqDecodingTest::DecodeAndCompare(const std::string &rtp_file,
                                         const std::string &ref_file) {
  OpenInputFile(rtp_file);

  std::string ref_out_file = "";
  if (ref_file.empty()) {
    ref_out_file = webrtc::test::OutputPath() + "neteq_out.pcm";
  }
  RefFiles ref_files(ref_file, ref_out_file);

  NETEQTEST_RTPpacket rtp;
  ASSERT_GT(rtp.readFromFile(rtp_fp_), 0);
  while (rtp.dataLen() >= 0) {
    int16_t out_len;
    Process(&rtp, &out_len);
    ref_files.ProcessReference(out_data_, out_len);
  }
}

void NetEqDecodingTest::DecodeAndCheckStats(const std::string &rtp_file,
                                            const std::string &stat_ref_file,
                                            const std::string &rtcp_ref_file) {
  OpenInputFile(rtp_file);
  std::string stat_out_file = "";
  if (stat_ref_file.empty()) {
    stat_out_file = webrtc::test::OutputPath() +
        "neteq_network_stats.dat";
  }
  RefFiles network_stat_files(stat_ref_file, stat_out_file);

  std::string rtcp_out_file = "";
  if (rtcp_ref_file.empty()) {
    rtcp_out_file = webrtc::test::OutputPath() +
        "neteq_rtcp_stats.dat";
  }
  RefFiles rtcp_stat_files(rtcp_ref_file, rtcp_out_file);

  NETEQTEST_RTPpacket rtp;
  ASSERT_GT(rtp.readFromFile(rtp_fp_), 0);
  while (rtp.dataLen() >= 0) {
    int16_t out_len;
    Process(&rtp, &out_len);

    // Query the network statistics API once per second
    if (sim_clock_ % 1000 == 0) {
      // Process NetworkStatistics.
      WebRtcNetEQ_NetworkStatistics network_stats;
      ASSERT_EQ(0, WebRtcNetEQ_GetNetworkStatistics(neteq_inst_->instance(),
                                                    &network_stats));
      network_stat_files.ProcessReference(network_stats);

      // Process RTCPstat.
      WebRtcNetEQ_RTCPStat rtcp_stats;
      ASSERT_EQ(0, WebRtcNetEQ_GetRTCPStats(neteq_inst_->instance(),
                                            &rtcp_stats));
      rtcp_stat_files.ProcessReference(rtcp_stats);
    }
  }
}

#if defined(WEBRTC_LINUX) && defined(WEBRTC_ARCH_64_BITS)
TEST_F(NetEqDecodingTest, TestBitExactness) {
  const std::string kInputRtpFile = webrtc::test::ProjectRootPath() +
      "test/data/audio_coding/universal.rtp";
  const std::string kInputRefFile = webrtc::test::ProjectRootPath() +
      "test/data/audio_coding/universal_ref.pcm";
  DecodeAndCompare(kInputRtpFile, kInputRefFile);
}
#endif  // defined(WEBRTC_LINUX) && defined(WEBRTC_ARCH_64_BITS)

//TODO(hlundin): Add test target NetEqDecodingTest::TestNetworkStatistics.

}  // namespace
<|endoftext|>