<commit_before>#include "def_lang/impl/Vector_Value.h"
#include "def_lang/Value_Selector.h"
#include "def_lang/Serialization.h"

namespace ts
{

Vector_Value::Vector_Value(std::shared_ptr<IVector_Type const> type)
    : m_type(type)
{
}

Result<bool> Vector_Value::is_equal(IValue const& other) const
{
    IVector_Value const* v = dynamic_cast<const IVector_Value*>(&other);
    if (!v)
    {
        return Error("incompatible values");
    }

    if (get_type() != v->get_type())
    {
        return Error("incompatible types");
    }

    if (get_value_count() != v->get_value_count())
    {
        return false;
    }

    for (size_t i = 0; i < get_value_count(); i++)
    {
        auto result = get_value(i)->is_equal(*v->get_value(i));
        if (result != success)
        {
            return result;
        }
        if (result.payload() == false)
        {
            return false;
        }
    }

    return true;
}

Result<void> Vector_Value::copy_assign(IValue const& other)
{
    IVector_Value const* v = dynamic_cast<const IVector_Value*>(&other);
    if (!v)
    {
        return Error("incompatible values");
    }

    if (get_type() != v->get_type())
    {
        return Error("incompatible types");
    }

    size_t common_size = std::min(get_value_count(), v->get_value_count());
    for (size_t i = 0; i < common_size; i++)
    {
        auto result = get_value(i)->copy_assign(*v->get_value(i));
        if (result != success)
        {
            return result;
        }
    }

    //insert missing values
    while (get_value_count() < v->get_value_count())
    {
        size_t idx = get_value_count();
        auto result = insert_value(idx, v->get_value(idx)->clone());
        if (result != success)
        {
            return result;
        }
    }

    //remove extra values
    while (get_value_count() > v->get_value_count())
    {
        size_t idx = get_value_count() - 1;
        auto result = erase_value(idx);
        if (result != success)
        {
            return result;
        }
    }

    return success;
}
Result<void> Vector_Value::copy_assign(IInitializer const& initializer)
{
    return Error("not implemented");
}

std::shared_ptr<IValue> Vector_Value::clone() const
{
    return std::make_shared<Vector_Value>(*this);
}

std::shared_ptr<IType const> Vector_Value::get_type() const
{
    return m_type;
}

Result<void> Vector_Value::parse_from_ui_string(std::string const& str)
{
    return Error("Not Supported");
}
Result<std::string> Vector_Value::get_ui_string() const
{
    return Error("Not Supported");
}

std::shared_ptr<const IValue> Vector_Value::select(Value_Selector&& selector) const
{
    return const_cast<Vector_Value*>(this)->select(std::move(selector));
}
std::shared_ptr<IValue> Vector_Value::select(Value_Selector&& selector)
{
    TS_ASSERT(!selector.empty());
    if (selector.empty())
    {
        return nullptr;
    }

    std::shared_ptr<IValue> value;
    Value_Selector::Element const& element = selector.front();
    if (element.type == Value_Selector::Element::Type::INDEX)
    {
        value = element.index < get_value_count() ? get_value(element.index) : std::shared_ptr<IValue>();
    }

    if (!value)
    {
        return nullptr;
    }

    selector.pop_front();
    if (selector.empty())
    {
        return value;
    }

    return value->select(std::move(selector));
}

std::shared_ptr<IVector_Type const> Vector_Value::get_specialized_type() const
{
    return m_type;
}

Result<serialization::Value> Vector_Value::serialize() const
{
    serialization::Value svalue(serialization::Value::Type::ARRAY);

    for (size_t i = 0; i < get_value_count(); i++)
    {
        auto result = get_value(i)->serialize();
        if (result != success)
        {
            return result;
        }
        svalue.add_array_element(result.extract_payload());
    }

    return std::move(svalue);
}

Result<void> Vector_Value::deserialize(serialization::Value const& sz_value)
{
    if (sz_value.get_type() != serialization::Value::Type::ARRAY)
    {
        return Error("Expected array value when deserializing");
    }

    while (get_value_count() > sz_value.get_array_element_count())
    {
        erase_value(get_value_count() - 1);
    }
    while (get_value_count() < sz_value.get_array_element_count())
    {
        insert_default_value(get_value_count());
    }

    for (size_t i = 0; i < sz_value.get_array_element_count(); i++)
    {
        auto result = get_value(i)->deserialize(sz_value.get_array_element_value(i));
        if (result != ts::success)
        {
            return result;
        }
    }

    return success;
}

Result<std::shared_ptr<IValue>> Vector_Value::insert_default_value(size_t idx)
{
    std::shared_ptr<IValue> value = get_specialized_type()->get_inner_type()->create_value();
    auto result = insert_value(idx, value);
    if (result != success)
    {
        return result.error();
    }
    return value;
}

Result<void> Vector_Value::insert_value(size_t idx, std::shared_ptr<IValue> value)
{
    if (!value)
    {
        return Error("Cannot insert null value");
    }

    if (idx > get_value_count())
    {
        return Error("Cannot insert beyond the end");
    }
    if (value->get_type() != get_specialized_type()->get_inner_type())
    {
        return Error("Cannot insert value of type '" + value->get_type()->get_name() + "'. Expected values of type '" + get_specialized_type()->get_inner_type()->get_name() + "'");
    }

    m_values.insert(m_values.begin() + idx, std::move(value));

    return success;
}
Result<void> Vector_Value::erase_value(size_t idx)
{
    if (idx >= get_value_count())
    {
        return Error("Cannot erase beyond the end");
    }

    m_values.erase(m_values.begin() + idx);

    return success;
}

void Vector_Value::clear()
{
    m_values.clear();
}

size_t Vector_Value::get_value_count() const
{
    return m_values.size();
}

std::shared_ptr<const IValue> Vector_Value::get_value(size_t idx) const
{
    return m_values[idx];
}
std::shared_ptr<IValue> Vector_Value::get_value(size_t idx)
{
    return m_values[idx];
}

}
<commit_msg>Fixed the deserialization of Vector_Value<commit_after>#include "def_lang/impl/Vector_Value.h"
#include "def_lang/Value_Selector.h"
#include "def_lang/Serialization.h"

namespace ts
{

Vector_Value::Vector_Value(std::shared_ptr<IVector_Type const> type)
    : m_type(type)
{
}

Result<bool> Vector_Value::is_equal(IValue const& other) const
{
    IVector_Value const* v = dynamic_cast<const IVector_Value*>(&other);
    if (!v)
    {
        return Error("incompatible values");
    }

    if (get_type() != v->get_type())
    {
        return Error("incompatible types");
    }

    if (get_value_count() != v->get_value_count())
    {
        return false;
    }

    for (size_t i = 0; i < get_value_count(); i++)
    {
        auto result = get_value(i)->is_equal(*v->get_value(i));
        if (result != success)
        {
            return result;
        }
        if (result.payload() == false)
        {
            return false;
        }
    }

    return true;
}

Result<void> Vector_Value::copy_assign(IValue const& other)
{
    IVector_Value const* v = dynamic_cast<const IVector_Value*>(&other);
    if (!v)
    {
        return Error("incompatible values");
    }

    if (get_type() != v->get_type())
    {
        return Error("incompatible types");
    }

    size_t common_size = std::min(get_value_count(), v->get_value_count());
    for (size_t i = 0; i < common_size; i++)
    {
        auto result = get_value(i)->copy_assign(*v->get_value(i));
        if (result != success)
        {
            return result;
        }
    }

    //insert missing values
    while (get_value_count() < v->get_value_count())
    {
        size_t idx = get_value_count();
        auto result = insert_value(idx, v->get_value(idx)->clone());
        if (result != success)
        {
            return result;
        }
    }

    //remove extra values
    while (get_value_count() > v->get_value_count())
    {
        size_t idx = get_value_count() - 1;
        auto result = erase_value(idx);
        if (result != success)
        {
            return result;
        }
    }

    return success;
}
Result<void> Vector_Value::copy_assign(IInitializer const& initializer)
{
    return Error("not implemented");
}

std::shared_ptr<IValue> Vector_Value::clone() const
{
    return std::make_shared<Vector_Value>(*this);
}

std::shared_ptr<IType const> Vector_Value::get_type() const
{
    return m_type;
}

Result<void> Vector_Value::parse_from_ui_string(std::string const& str)
{
    return Error("Not Supported");
}
Result<std::string> Vector_Value::get_ui_string() const
{
    return Error("Not Supported");
}

std::shared_ptr<const IValue> Vector_Value::select(Value_Selector&& selector) const
{
    return const_cast<Vector_Value*>(this)->select(std::move(selector));
}
std::shared_ptr<IValue> Vector_Value::select(Value_Selector&& selector)
{
    TS_ASSERT(!selector.empty());
    if (selector.empty())
    {
        return nullptr;
    }

    std::shared_ptr<IValue> value;
    Value_Selector::Element const& element = selector.front();
    if (element.type == Value_Selector::Element::Type::INDEX)
    {
        value = element.index < get_value_count() ? get_value(element.index) : std::shared_ptr<IValue>();
    }

    if (!value)
    {
        return nullptr;
    }

    selector.pop_front();
    if (selector.empty())
    {
        return value;
    }

    return value->select(std::move(selector));
}

std::shared_ptr<IVector_Type const> Vector_Value::get_specialized_type() const
{
    return m_type;
}

Result<serialization::Value> Vector_Value::serialize() const
{
    serialization::Value svalue(serialization::Value::Type::ARRAY);

    for (size_t i = 0; i < get_value_count(); i++)
    {
        auto result = get_value(i)->serialize();
        if (result != success)
        {
            return result;
        }
        svalue.add_array_element(result.extract_payload());
    }

    return std::move(svalue);
}

Result<void> Vector_Value::deserialize(serialization::Value const& sz_value)
{
    if (sz_value.get_type() != serialization::Value::Type::ARRAY)
    {
        return Error("Expected array value when deserializing");
    }

    while (get_value_count() > sz_value.get_array_element_count())
    {
        auto result = erase_value(get_value_count() - 1);
        TS_ASSERT(result == success);
    }
    while (get_value_count() < sz_value.get_array_element_count())
    {
        auto result = insert_default_value(get_value_count());
        TS_ASSERT(result == success);
    }

    for (size_t i = 0; i < sz_value.get_array_element_count(); i++)
    {
        auto result = get_value(i)->deserialize(sz_value.get_array_element_value(i));
        if (result != ts::success)
        {
            return result;
        }
    }

    return success;
}

Result<std::shared_ptr<IValue>> Vector_Value::insert_default_value(size_t idx)
{
    std::shared_ptr<IValue> value = get_specialized_type()->get_inner_type()->create_value();
    auto result = insert_value(idx, value);
    if (result != success)
    {
        return result.error();
    }
    return value;
}

Result<void> Vector_Value::insert_value(size_t idx, std::shared_ptr<IValue> value)
{
    if (!value)
    {
        return Error("Cannot insert null value");
    }

    if (idx > get_value_count())
    {
        return Error("Cannot insert beyond the end");
    }
    if (value->get_type() != get_specialized_type()->get_inner_type())
    {
        return Error("Cannot insert value of type '" + value->get_type()->get_name() + "'. Expected values of type '" + get_specialized_type()->get_inner_type()->get_name() + "'");
    }

    m_values.insert(m_values.begin() + idx, std::move(value));

    return success;
}
Result<void> Vector_Value::erase_value(size_t idx)
{
    if (idx >= get_value_count())
    {
        return Error("Cannot erase beyond the end");
    }

    m_values.erase(m_values.begin() + idx);

    return success;
}

void Vector_Value::clear()
{
    m_values.clear();
}

size_t Vector_Value::get_value_count() const
{
    return m_values.size();
}

std::shared_ptr<const IValue> Vector_Value::get_value(size_t idx) const
{
    return m_values[idx];
}
std::shared_ptr<IValue> Vector_Value::get_value(size_t idx)
{
    return m_values[idx];
}

}
<|endoftext|>


<commit_before>#include "SemposScorer.h"

#include <algorithm>
#include <vector>
#include <stdexcept>
#include <fstream>

#include "Util.h"
#include "SemposOverlapping.h"

using namespace std;

SemposScorer::SemposScorer(const string& config)
  : StatisticsBasedScorer("SEMPOS", config),
    m_ovr(SemposOverlappingFactory::GetOverlapping(getConfig("overlapping", "cap-micro"),this)),
    m_enable_debug(false)
{
  const string& debugSwitch = getConfig("debug", "0");
  if (debugSwitch == "1") m_enable_debug = true;

  m_semposMap.clear();

  string weightsfile = getConfig("weightsfile", "");
  if (weightsfile != "")
  {
    loadWeights(weightsfile);
  }
}

SemposScorer::~SemposScorer() {}

void SemposScorer::setReferenceFiles(const vector<string>& referenceFiles)
{
  //make sure reference data is clear
  m_ref_sentences.clear();

  //load reference data
  for (size_t rid = 0; rid < referenceFiles.size(); ++rid) {
    ifstream refin(referenceFiles[rid].c_str());
    if (!refin) {
      throw runtime_error("Unable to open: " + referenceFiles[rid]);
    }
    m_ref_sentences.push_back(vector<sentence_t>());
    string line;
    while (getline(refin,line)) {
      line = preprocessSentence(line);

      str_sentence_t sentence;
      splitSentence(line, sentence);

      sentence_t encodedSentence;
      encodeSentence(sentence, encodedSentence);

      m_ref_sentences[rid].push_back(encodedSentence);
    }
  }
}

void SemposScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
{
  vector<ScoreStatsType> stats;

  const string& sentence = preprocessSentence(text);
  str_sentence_t splitCandSentence;
  splitSentence(sentence, splitCandSentence);

  sentence_t encodedCandSentence;
  encodeSentence(splitCandSentence, encodedCandSentence);

  if (m_ref_sentences.size() == 1) {
    stats = m_ovr->prepareStats(encodedCandSentence, m_ref_sentences[0][sid]);
  } else {
    float max = -1.0f;
    for (size_t rid = 0; rid < m_ref_sentences.size(); ++rid) {
      const vector<ScoreStatsType>& tmp = m_ovr->prepareStats(encodedCandSentence, m_ref_sentences[rid][sid]);
      if (m_ovr->calculateScore(tmp) > max) {
        stats = tmp;
      }
    }
  }
  entry.set(stats);
}

void SemposScorer::splitSentence(const string& sentence, str_sentence_t& splitSentence)
{
  splitSentence.clear();

  vector<string> tokens;
  split(sentence, ' ', tokens);
  for (vector<string>::iterator it = tokens.begin(); it != tokens.end(); ++it) {
    vector<string> factors;
    split(*it, '|', factors);
    if (factors.size() != 2) throw runtime_error("Sempos scorer accepts two factors (item|class)");
    const string& item = factors[0];
    const string& klass = factors[1];
    splitSentence.push_back(make_pair(item, klass));
  }
}

void SemposScorer::encodeSentence(const str_sentence_t& sentence, sentence_t& encodedSentence)
{
  for (str_sentence_it it = sentence.begin(); it != sentence.end(); ++it) {
    const int tlemma = encodeString(it->first);
    const int sempos = encodeSempos(it->second);
    if (sempos >= 0) {
      encodedSentence.insert(make_pair(tlemma,sempos));
    }
  }
}

int SemposScorer::encodeString(const string& str)
{
  encoding_it encoding = m_stringMap.find(str);
  int encoded_str;
  if (encoding == m_stringMap.end()) {
    encoded_str = static_cast<int>(m_stringMap.size());
    m_stringMap[str] = encoded_str;
  } else {
    encoded_str = encoding->second;
  }
  return encoded_str;
}

int SemposScorer::encodeSempos(const string& sempos)
{
  if (sempos == "-") return -1;
  encoding_it it = m_semposMap.find(sempos);
  if (it == m_semposMap.end()) {
    const int classNumber = static_cast<int>(m_semposMap.size());
    if (classNumber == kMaxNOC) {
      throw std::runtime_error("Number of classes is greater than kMaxNOC");
    }
    m_semposMap[sempos] = classNumber;
    return classNumber;
  } else {
    return it->second;
  }
}

float SemposScorer::weight(int item) const
{
    std::map<int,float>::const_iterator it = weightsMap.find(item);
    if (it == weightsMap.end())
    {
        return 1.0f;
    }
    else
    {
        return it->second;
    }
}

void SemposScorer::loadWeights(const string& weightsfile)
{
    string line;
    ifstream myfile;
    myfile.open(weightsfile.c_str(), ifstream::in);
    if (myfile.is_open())
    {
        while ( myfile.good() )
        {
            getline (myfile,line);
            vector<string> fields;
            if (line == "") continue;
            split(line, '\t', fields);
            if (fields.size() != 2) throw std::runtime_error("Bad format of a row in weights file.");
            int encoded = encodeString(fields[0]);
            float weight = atof(fields[1].c_str());
            weightsMap[encoded] = weight;
        }
        myfile.close();
    }
    else
    {
        cerr << "Unable to open file "<< weightsfile << endl;
        exit(1);
    }

}
<commit_msg>Fixed bug in SemposScorer.cpp<commit_after>#include "SemposScorer.h"

#include <algorithm>
#include <vector>
#include <stdexcept>
#include <fstream>

#include "Util.h"
#include "SemposOverlapping.h"

using namespace std;

SemposScorer::SemposScorer(const string& config)
  : StatisticsBasedScorer("SEMPOS", config),
    m_ovr(SemposOverlappingFactory::GetOverlapping(getConfig("overlapping", "cap-micro"),this)),
    m_enable_debug(false)
{
  const string& debugSwitch = getConfig("debug", "0");
  if (debugSwitch == "1") m_enable_debug = true;

  m_semposMap.clear();

  string weightsfile = getConfig("weightsfile", "");
  if (weightsfile != "")
  {
    loadWeights(weightsfile);
  }
}

SemposScorer::~SemposScorer() {}

void SemposScorer::setReferenceFiles(const vector<string>& referenceFiles)
{
  //make sure reference data is clear
  m_ref_sentences.clear();

  //load reference data
  for (size_t rid = 0; rid < referenceFiles.size(); ++rid) {
    ifstream refin(referenceFiles[rid].c_str());
    if (!refin) {
      throw runtime_error("Unable to open: " + referenceFiles[rid]);
    }
    m_ref_sentences.push_back(vector<sentence_t>());
    string line;
    while (getline(refin,line)) {
      line = preprocessSentence(line);

      str_sentence_t sentence;
      splitSentence(line, sentence);

      sentence_t encodedSentence;
      encodeSentence(sentence, encodedSentence);

      m_ref_sentences[rid].push_back(encodedSentence);
    }
  }
}

void SemposScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
{
  vector<ScoreStatsType> stats;

  const string& sentence = preprocessSentence(text);
  str_sentence_t splitCandSentence;
  splitSentence(sentence, splitCandSentence);

  sentence_t encodedCandSentence;
  encodeSentence(splitCandSentence, encodedCandSentence);

  if (m_ref_sentences.size() == 1) {
    stats = m_ovr->prepareStats(encodedCandSentence, m_ref_sentences[0][sid]);
  } else {
    float max = -1.0f;
    for (size_t rid = 0; rid < m_ref_sentences.size(); ++rid) {
      const vector<ScoreStatsType>& tmp = m_ovr->prepareStats(encodedCandSentence, m_ref_sentences[rid][sid]);
      if (m_ovr->calculateScore(tmp) > max) {
        stats = tmp;
      }
    }
  }
  entry.set(stats);
}

void SemposScorer::splitSentence(const string& sentence, str_sentence_t& splitSentence)
{
  splitSentence.clear();

  vector<string> tokens;
  split(sentence, ' ', tokens);
  for (vector<string>::iterator it = tokens.begin(); it != tokens.end(); ++it) {
    vector<string> factors;
    if (it->empty()) continue;
    split(*it, '|', factors);
    if (factors.size() != 2) throw runtime_error("Sempos scorer accepts two factors (item|class)");
    const string& item = factors[0];
    const string& klass = factors[1];
    splitSentence.push_back(make_pair(item, klass));
  }
}

void SemposScorer::encodeSentence(const str_sentence_t& sentence, sentence_t& encodedSentence)
{
  for (str_sentence_it it = sentence.begin(); it != sentence.end(); ++it) {
    const int tlemma = encodeString(it->first);
    const int sempos = encodeSempos(it->second);
    if (sempos >= 0) {
      encodedSentence.insert(make_pair(tlemma,sempos));
    }
  }
}

int SemposScorer::encodeString(const string& str)
{
  encoding_it encoding = m_stringMap.find(str);
  int encoded_str;
  if (encoding == m_stringMap.end()) {
    encoded_str = static_cast<int>(m_stringMap.size());
    m_stringMap[str] = encoded_str;
  } else {
    encoded_str = encoding->second;
  }
  return encoded_str;
}

int SemposScorer::encodeSempos(const string& sempos)
{
  if (sempos == "-") return -1;
  encoding_it it = m_semposMap.find(sempos);
  if (it == m_semposMap.end()) {
    const int classNumber = static_cast<int>(m_semposMap.size());
    if (classNumber == kMaxNOC) {
      throw std::runtime_error("Number of classes is greater than kMaxNOC");
    }
    m_semposMap[sempos] = classNumber;
    return classNumber;
  } else {
    return it->second;
  }
}

float SemposScorer::weight(int item) const
{
    std::map<int,float>::const_iterator it = weightsMap.find(item);
    if (it == weightsMap.end())
    {
        return 1.0f;
    }
    else
    {
        return it->second;
    }
}

void SemposScorer::loadWeights(const string& weightsfile)
{
    string line;
    ifstream myfile;
    myfile.open(weightsfile.c_str(), ifstream::in);
    if (myfile.is_open())
    {
        while ( myfile.good() )
        {
            getline (myfile,line);
            vector<string> fields;
            if (line == "") continue;
            split(line, '\t', fields);
            if (fields.size() != 2) throw std::runtime_error("Bad format of a row in weights file.");
            int encoded = encodeString(fields[0]);
            float weight = atof(fields[1].c_str());
            weightsMap[encoded] = weight;
        }
        myfile.close();
    }
    else
    {
        cerr << "Unable to open file "<< weightsfile << endl;
        exit(1);
    }

}
<|endoftext|>