<commit_before>extern "C" {
#include <sys/uio.h>
#include <unistd.h>
}
#include "start_thread.h"
#include "tmd.h"
#include "incline_def_async_qtable.h"
#include "incline_driver_async_qtable.h"
#include "incline_mgr.h"
#include "incline_util.h"

using namespace std;

incline_def*
incline_driver_async_qtable::create_def() const
{
  return new incline_def_async_qtable();
}

vector<string>
incline_driver_async_qtable::create_table_all(bool if_not_exists,
					      tmd::conn_t& dbh) const
{
  vector<string> r;
  for (std::vector<incline_def*>::const_iterator di = mgr_->defs().begin();
       di != mgr_->defs().end();
       ++di) {
    r.push_back(create_table_of(*di, if_not_exists, dbh));
  }
  return r;
}

vector<string>
incline_driver_async_qtable::drop_table_all(bool if_exists) const
{
  vector<string> r;
  for (std::vector<incline_def*>::const_iterator di = mgr_->defs().begin();
       di != mgr_->defs().end();
       ++di) {
    r.push_back(drop_table_of(*di, if_exists));
  }
  return r;
}

string
incline_driver_async_qtable::create_table_of(const incline_def* _def,
					     bool if_not_exists,
					     tmd::conn_t& dbh) const
{
  const incline_def_async_qtable* def
    = dynamic_cast<const incline_def_async_qtable*>(_def);
  assert(def != NULL);
  return _create_table_of(def, def->queue_table(), if_not_exists, dbh);
}

string
incline_driver_async_qtable::drop_table_of(const incline_def* _def,
					   bool if_exists) const
{
  const incline_def_async_qtable* def
    = dynamic_cast<const incline_def_async_qtable*>(_def);
  assert(def != NULL);
  return string("DROP TABLE ") + (if_exists ? "IF EXISTS " : "")
    + def->queue_table();
}

string
incline_driver_async_qtable::_create_table_of(const incline_def_async_qtable*
					      def,
					      const std::string& table_name,
					      bool if_not_exists,
					      tmd::conn_t& dbh) const
{
  vector<string> col_defs;
  for (map<string, string>::const_iterator ci = def->columns().begin();
       ci != def->columns().end();
       ++ci) {
    tmd::query_t res(dbh,
		     "SELECT UPPER(COLUMN_TYPE),CHARACTER_SET_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s' AND COLUMN_NAME='%s'",
		     tmd::escape(dbh, mgr_->db_name()).c_str(),
		     tmd::escape(dbh, def->destination()).c_str(),
		     tmd::escape(dbh, ci->second).c_str());
    if (res.fetch().eof()) {
      // TODO throw an exception instead
      cerr << "failed to obtain column definition of: " << ci->first << endl;
      exit(4);
    }
    col_defs.push_back(ci->second + ' ' + res.field(0));
    if (res.field(1) != NULL) {
      col_defs.back() += string(" CHARSET ") + res.field(1);
    }
    col_defs.back() += " NOT NULL";
  }
  return string("CREATE TABLE ") + (if_not_exists ? "IF NOT EXISTS " : "")
    + table_name
    + (" (_iq_id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,"
       " _iq_action CHAR(1) CHARACTER SET latin1 NOT NULL,")
    + incline_util::join(',', col_defs)
    + ",PRIMARY KEY (_iq_id)) ENGINE InnoDB";
}

vector<string>
incline_driver_async_qtable::do_build_enqueue_insert_sql(const incline_def*
							 _def,
							 const string&
							 src_table,
							 const string& command,
							 const vector<string>*
							 cond)
  const
{
  const incline_def_async_qtable* def
    = dynamic_cast<const incline_def_async_qtable*>(_def);
  map<string, string> extra_columns;
  extra_columns["_iq_action"] = "'R'";
  string sql
    = incline_driver_standalone::_build_insert_from_def(def, def->queue_table(),
							src_table, command,
							cond, &extra_columns);
  return incline_util::vectorize(sql);
}

vector<string>
incline_driver_async_qtable::do_build_enqueue_delete_sql(const incline_def*
							 _def,
							 const string&
							 src_table,
							 const vector<string>*
							 _cond)
  const
{
  const incline_def_async_qtable* def
    = dynamic_cast<const incline_def_async_qtable*>(_def);
  map<string, string> pk_columns;
  for (map<string, string>::const_iterator pi = def->pk_columns().begin();
       pi != def->pk_columns().end();
       ++pi) {
    string src = pi->first;
    if (incline_def::table_of_column(pi->first) == src_table) {
      src = "OLD" + src.substr(src_table.size());
    }
    pk_columns[src] = pi->second;
  }
  vector<string> tables;
  for (vector<string>::const_iterator si = def->source().begin();
       si != def->source().end();
       ++si) {
    if (*si != src_table && def->is_master_of(*si)) {
      tables.push_back(*si);
    }
  }
  vector<string> cond = def->build_merge_cond(src_table, "OLD", true);
  if (_cond != NULL) {
    incline_util::push_back(cond, *_cond);
  }
  string sql = "INSERT INTO " + def->queue_table() + " ("
    + incline_util::join(',', incline_util::filter("%2", pk_columns))
    + ",_iq_action) SELECT "
    + incline_util::join(',', incline_util::filter("%1", pk_columns))
    + ",'D'";
  if (! tables.empty()) {
    sql += " FROM " + incline_util::join(" INNER JOIN ", tables);
  }
  if (! cond.empty()){
    sql += " WHERE " + incline_util::join(" AND ", cond);
  }
  return incline_util::vectorize(sql);
}

incline_driver_async_qtable::forwarder::forwarder(forwarder_mgr* mgr,
						  const
						  incline_def_async_qtable* def,
						  tmd::conn_t* dbh,
						  int poll_interval)
  : mgr_(mgr), def_(def), dbh_(dbh), poll_interval_(poll_interval),
    dest_pk_columns_(incline_util::filter("%2", def->pk_columns()))
{
  fetch_query_base_ = "SELECT _iq_id,_iq_action,"
    + incline_util::join(',',
			 incline_util::filter("%2", def_->pk_columns()));
  if (! def_->npk_columns().empty()) {
    fetch_query_base_ += ','
      + incline_util::join(',',
			   incline_util::filter("%2",
						def_->npk_columns()));
  }
  fetch_query_base_ += " FROM " + def_->queue_table() + ' ';
  clear_queue_query_base_ = "DELETE FROM " + def_->queue_table()
    + " WHERE _iq_id IN ";
  // build write queries
  {
    vector<string> dest_cols(incline_util::filter("%2", def->pk_columns()));
    incline_util::push_back(dest_cols,
			    incline_util::filter("%2", def->npk_columns()));
    replace_row_query_base_ =
      "REPLACE INTO " + def->destination() + " ("
      + incline_util::join(',', dest_cols) + ") VALUES ";
  }
  delete_row_query_base_ = "DELETE FROM " + def->destination() + " WHERE ";
}

incline_driver_async_qtable::forwarder::~forwarder()
{
  delete dbh_;
}

void* incline_driver_async_qtable::forwarder::run()
{
  while (1) {
    try {
      vector<string> iq_ids;
      vector<pair<char, vector<string> > > rows;
      { // fetch data
	string query = fetch_query_base_;
	string extra_cond = do_get_extra_cond();
	if (! extra_cond.empty()) {
	  // TODO create and use index shard_key,_iq_id
	  query += " WHERE " + extra_cond;
	}
	query += " ORDER BY _iq_id LIMIT 50";
	// load rows
	for (tmd::query_t res(*dbh_, query);
	     ! res.fetch().eof();
	     ) {
	  iq_ids.push_back(res.field(0));
	  char action = res.field(1)[0];
	  rows.push_back(make_pair(action, vector<string>()));
	  for (size_t i = 0;
	       i < (action == 'R'
		    ? def_->columns().size() : def_->pk_columns().size());
	       ++i) {
	    rows.back().second.push_back(res.field(i + 2));
	  }
	}
      }
      // sleep and retry if no data
      if (rows.empty()) {
	sleep(poll_interval_);
	continue;
      }
      vector<const vector<string>*> replace_rows, delete_pks;
      // fill replace_rows and delete_rows
      for (vector<pair<char, vector<string> > >::const_reverse_iterator ri
	     = rows.rbegin();
	   ri != rows.rend();
	   ++ri) {
	for (vector<pair<char, vector<string> > >::const_reverse_iterator ci
	       = rows.rbegin();
	     ci != ri;
	     ++ci) {
	  for (size_t i = 0; i < def_->pk_columns().size(); ++i) {
	    if (ri->second[i] != ci->second[i]) {
	      goto ROW_NOT_EQUAL;
	    }
	  }
	  goto EQ_ROW_FOUND;
	ROW_NOT_EQUAL:
	  ;
	}
	// row with same pk not exists, register it
	switch (ri->first) {
	case 'R': // replace
	  replace_rows.push_back(&ri->second);
	  break;
	case 'D': // delete
	  delete_pks.push_back(&ri->second);
	  break;
	default:
	  assert(0);
	}
      EQ_ROW_FOUND:
	;
      }
      // update and remove from queue if successful
      if (do_update_rows(replace_rows, delete_pks)) {
	tmd::execute(*dbh_,
		     clear_queue_query_base_ + '('
		     + incline_util::join(',', iq_ids) + ')');
      }
    } catch (tmd::error_t& e) {
      switch (e.mysql_errno()) {
      case ER_LOCK_DEADLOCK:
      case ER_LOCK_WAIT_TIMEOUT:
	// just retry
	break;
      default:
	throw;
      }
    }
  }

  return NULL;
}

bool
incline_driver_async_qtable::forwarder::do_update_rows(const vector<const vector<string>*>& replace_rows, const vector<const vector<string>*>& delete_rows)
{
  if (! replace_rows.empty()) {
    this->replace_rows(*dbh_, replace_rows);
  }
  if (! delete_rows.empty()) {
    this->delete_rows(*dbh_, delete_rows);
  }
  return true;
}

string
incline_driver_async_qtable::forwarder::do_get_extra_cond()
{
  return string();
}

void
incline_driver_async_qtable::forwarder::replace_rows(tmd::conn_t& dbh,
						     const vector<const vector<string>*>& rows) const
{
  string sql = replace_row_query_base_ + '(';
  for (vector<const vector<string>*>::const_iterator ri = rows.begin();
       ri != rows.end();
       ++ri) {
    for (vector<string>::const_iterator ci = (*ri)->begin();
	 ci != (*ri)->end();
	 ++ci) {
      sql.push_back('\'');
      sql += tmd::escape(dbh, *ci);
      sql += "',";
    }
    sql.erase(sql.size() - 1);
    sql += "),(";
  }
  sql.erase(sql.size() - 2);
  mgr_->log_sql(sql);
  tmd::execute(dbh, sql);
}

void
incline_driver_async_qtable::forwarder::delete_rows(tmd::conn_t& dbh,
						    const vector<const vector<string>*>& pk_rows) const
{
  vector<string> conds;
  for (vector<const vector<string>*>::const_iterator pi = pk_rows.begin();
       pi != pk_rows.end();
       ++pi) {
    conds.push_back("(" + _build_pk_cond(dbh, dest_pk_columns_, **pi) + ')');
  }
  string sql = delete_row_query_base_ + incline_util::join(" OR ", conds);
  mgr_->log_sql(sql);
  tmd::execute(dbh, sql);
}

string
incline_driver_async_qtable::forwarder::_build_pk_cond(tmd::conn_t& dbh,
						       const vector<string>&
						       colnames,
						       const vector<string>&
						       rows)
{
  assert(colnames.size() == rows.size());
  vector<string> cond;
  for (size_t i = 0; i < rows.size(); ++i) {
    cond.push_back(colnames[i] + "='" + tmd::escape(dbh, rows[i]) + '\'');
  }
  return incline_util::join(" AND ", cond);
}

void*
incline_driver_async_qtable::forwarder_mgr::run()
{
  vector<pthread_t> threads;

  { // create and start forwarders
    const vector<incline_def*>& defs = driver()->mgr()->defs();
    for (vector<incline_def*>::const_iterator di = defs.begin();
	 di != defs.end();
	 ++di) {
      const incline_def_async_qtable* def
	= dynamic_cast<const incline_def_async_qtable*>(*di);
      assert(def != NULL);
      threads.push_back(start_thread(do_create_forwarder(def)));
    }
  }

  // loop
  while (! threads.empty()) {
    pthread_join(threads.back(), NULL);
    threads.pop_back();
  }

  return NULL;
}

void
incline_driver_async_qtable::forwarder_mgr::log_sql(const string& sql)
{
  if (log_fd_ != -1) {
    struct iovec vec[2];
    vec[0].iov_base = const_cast<char*>(sql.c_str());
    vec[0].iov_len = sql.size();
    vec[1].iov_base = const_cast<char*>(";\n");
    vec[1].iov_len = 2;
    writev(log_fd_, vec, 2);
  }
}

incline_driver_async_qtable::forwarder*
incline_driver_async_qtable::forwarder_mgr::do_create_forwarder(const incline_def_async_qtable* def)
{
  tmd::conn_t* dbh = (*connect_)(src_host_.c_str(), src_port_);
  assert(dbh != NULL);
  return new forwarder(this, def, dbh, poll_interval_);
}
<commit_msg>proper handling of disconn. of shards (untested)<commit_after>extern "C" {
#include <sys/uio.h>
#include <unistd.h>
}
#include "start_thread.h"
#include "tmd.h"
#include "incline_def_async_qtable.h"
#include "incline_driver_async_qtable.h"
#include "incline_mgr.h"
#include "incline_util.h"

using namespace std;

incline_def*
incline_driver_async_qtable::create_def() const
{
  return new incline_def_async_qtable();
}

vector<string>
incline_driver_async_qtable::create_table_all(bool if_not_exists,
					      tmd::conn_t& dbh) const
{
  vector<string> r;
  for (std::vector<incline_def*>::const_iterator di = mgr_->defs().begin();
       di != mgr_->defs().end();
       ++di) {
    r.push_back(create_table_of(*di, if_not_exists, dbh));
  }
  return r;
}

vector<string>
incline_driver_async_qtable::drop_table_all(bool if_exists) const
{
  vector<string> r;
  for (std::vector<incline_def*>::const_iterator di = mgr_->defs().begin();
       di != mgr_->defs().end();
       ++di) {
    r.push_back(drop_table_of(*di, if_exists));
  }
  return r;
}

string
incline_driver_async_qtable::create_table_of(const incline_def* _def,
					     bool if_not_exists,
					     tmd::conn_t& dbh) const
{
  const incline_def_async_qtable* def
    = dynamic_cast<const incline_def_async_qtable*>(_def);
  assert(def != NULL);
  return _create_table_of(def, def->queue_table(), if_not_exists, dbh);
}

string
incline_driver_async_qtable::drop_table_of(const incline_def* _def,
					   bool if_exists) const
{
  const incline_def_async_qtable* def
    = dynamic_cast<const incline_def_async_qtable*>(_def);
  assert(def != NULL);
  return string("DROP TABLE ") + (if_exists ? "IF EXISTS " : "")
    + def->queue_table();
}

string
incline_driver_async_qtable::_create_table_of(const incline_def_async_qtable*
					      def,
					      const std::string& table_name,
					      bool if_not_exists,
					      tmd::conn_t& dbh) const
{
  vector<string> col_defs;
  for (map<string, string>::const_iterator ci = def->columns().begin();
       ci != def->columns().end();
       ++ci) {
    tmd::query_t res(dbh,
		     "SELECT UPPER(COLUMN_TYPE),CHARACTER_SET_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s' AND COLUMN_NAME='%s'",
		     tmd::escape(dbh, mgr_->db_name()).c_str(),
		     tmd::escape(dbh, def->destination()).c_str(),
		     tmd::escape(dbh, ci->second).c_str());
    if (res.fetch().eof()) {
      // TODO throw an exception instead
      cerr << "failed to obtain column definition of: " << ci->first << endl;
      exit(4);
    }
    col_defs.push_back(ci->second + ' ' + res.field(0));
    if (res.field(1) != NULL) {
      col_defs.back() += string(" CHARSET ") + res.field(1);
    }
    col_defs.back() += " NOT NULL";
  }
  return string("CREATE TABLE ") + (if_not_exists ? "IF NOT EXISTS " : "")
    + table_name
    + (" (_iq_id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,"
       " _iq_action CHAR(1) CHARACTER SET latin1 NOT NULL,")
    + incline_util::join(',', col_defs)
    + ",PRIMARY KEY (_iq_id)) ENGINE InnoDB";
}

vector<string>
incline_driver_async_qtable::do_build_enqueue_insert_sql(const incline_def*
							 _def,
							 const string&
							 src_table,
							 const string& command,
							 const vector<string>*
							 cond)
  const
{
  const incline_def_async_qtable* def
    = dynamic_cast<const incline_def_async_qtable*>(_def);
  map<string, string> extra_columns;
  extra_columns["_iq_action"] = "'R'";
  string sql
    = incline_driver_standalone::_build_insert_from_def(def, def->queue_table(),
							src_table, command,
							cond, &extra_columns);
  return incline_util::vectorize(sql);
}

vector<string>
incline_driver_async_qtable::do_build_enqueue_delete_sql(const incline_def*
							 _def,
							 const string&
							 src_table,
							 const vector<string>*
							 _cond)
  const
{
  const incline_def_async_qtable* def
    = dynamic_cast<const incline_def_async_qtable*>(_def);
  map<string, string> pk_columns;
  for (map<string, string>::const_iterator pi = def->pk_columns().begin();
       pi != def->pk_columns().end();
       ++pi) {
    string src = pi->first;
    if (incline_def::table_of_column(pi->first) == src_table) {
      src = "OLD" + src.substr(src_table.size());
    }
    pk_columns[src] = pi->second;
  }
  vector<string> tables;
  for (vector<string>::const_iterator si = def->source().begin();
       si != def->source().end();
       ++si) {
    if (*si != src_table && def->is_master_of(*si)) {
      tables.push_back(*si);
    }
  }
  vector<string> cond = def->build_merge_cond(src_table, "OLD", true);
  if (_cond != NULL) {
    incline_util::push_back(cond, *_cond);
  }
  string sql = "INSERT INTO " + def->queue_table() + " ("
    + incline_util::join(',', incline_util::filter("%2", pk_columns))
    + ",_iq_action) SELECT "
    + incline_util::join(',', incline_util::filter("%1", pk_columns))
    + ",'D'";
  if (! tables.empty()) {
    sql += " FROM " + incline_util::join(" INNER JOIN ", tables);
  }
  if (! cond.empty()){
    sql += " WHERE " + incline_util::join(" AND ", cond);
  }
  return incline_util::vectorize(sql);
}

incline_driver_async_qtable::forwarder::forwarder(forwarder_mgr* mgr,
						  const
						  incline_def_async_qtable* def,
						  tmd::conn_t* dbh,
						  int poll_interval)
  : mgr_(mgr), def_(def), dbh_(dbh), poll_interval_(poll_interval),
    dest_pk_columns_(incline_util::filter("%2", def->pk_columns()))
{
  fetch_query_base_ = "SELECT _iq_id,_iq_action,"
    + incline_util::join(',',
			 incline_util::filter("%2", def_->pk_columns()));
  if (! def_->npk_columns().empty()) {
    fetch_query_base_ += ','
      + incline_util::join(',',
			   incline_util::filter("%2",
						def_->npk_columns()));
  }
  fetch_query_base_ += " FROM " + def_->queue_table() + ' ';
  clear_queue_query_base_ = "DELETE FROM " + def_->queue_table()
    + " WHERE _iq_id IN ";
  // build write queries
  {
    vector<string> dest_cols(incline_util::filter("%2", def->pk_columns()));
    incline_util::push_back(dest_cols,
			    incline_util::filter("%2", def->npk_columns()));
    replace_row_query_base_ =
      "REPLACE INTO " + def->destination() + " ("
      + incline_util::join(',', dest_cols) + ") VALUES ";
  }
  delete_row_query_base_ = "DELETE FROM " + def->destination() + " WHERE ";
}

incline_driver_async_qtable::forwarder::~forwarder()
{
  delete dbh_;
}

void* incline_driver_async_qtable::forwarder::run()
{
  string extra_cond, last_id;

  while (1) {
    try {
      vector<string> iq_ids;
      vector<pair<char, vector<string> > > rows;
      { // update fetch state
	string new_extra_cond = do_get_extra_cond();
	if (extra_cond != new_extra_cond) {
	  extra_cond = new_extra_cond;
	  last_id.clear();
	}
      }
      { // fetch data
	string query = fetch_query_base_;
	if (! extra_cond.empty()) {
	  // TODO create and use index shard_key,_iq_id
	  query += " WHERE " + extra_cond;
	  if (! last_id.empty()) {
	    query += " AND _iq_id>" + last_id;
	  }
	}
	query += " ORDER BY _iq_id LIMIT 50";
	// load rows
	for (tmd::query_t res(*dbh_, query);
	     ! res.fetch().eof();
	     ) {
	  iq_ids.push_back(res.field(0));
	  char action = res.field(1)[0];
	  rows.push_back(make_pair(action, vector<string>()));
	  for (size_t i = 0;
	       i < (action == 'R'
		    ? def_->columns().size() : def_->pk_columns().size());
	       ++i) {
	    rows.back().second.push_back(res.field(i + 2));
	  }
	}
      }
      // sleep and retry if no data
      if (rows.empty()) {
	sleep(poll_interval_);
	continue;
      }
      if (! extra_cond.empty()) {
	last_id = iq_ids.back();
      }
      vector<const vector<string>*> replace_rows, delete_pks;
      // fill replace_rows and delete_rows
      for (vector<pair<char, vector<string> > >::const_reverse_iterator ri
	     = rows.rbegin();
	   ri != rows.rend();
	   ++ri) {
	for (vector<pair<char, vector<string> > >::const_reverse_iterator ci
	       = rows.rbegin();
	     ci != ri;
	     ++ci) {
	  for (size_t i = 0; i < def_->pk_columns().size(); ++i) {
	    if (ri->second[i] != ci->second[i]) {
	      goto ROW_NOT_EQUAL;
	    }
	  }
	  goto EQ_ROW_FOUND;
	ROW_NOT_EQUAL:
	  ;
	}
	// row with same pk not exists, register it
	switch (ri->first) {
	case 'R': // replace
	  replace_rows.push_back(&ri->second);
	  break;
	case 'D': // delete
	  delete_pks.push_back(&ri->second);
	  break;
	default:
	  assert(0);
	}
      EQ_ROW_FOUND:
	;
      }
      // update and remove from queue if successful
      if (do_update_rows(replace_rows, delete_pks)) {
	tmd::execute(*dbh_,
		     clear_queue_query_base_ + '('
		     + incline_util::join(',', iq_ids) + ')');
      }
    } catch (tmd::error_t& e) {
      switch (e.mysql_errno()) {
      case ER_LOCK_DEADLOCK:
      case ER_LOCK_WAIT_TIMEOUT:
	// just retry
	break;
      default:
	throw;
      }
    }
  }

  return NULL;
}

bool
incline_driver_async_qtable::forwarder::do_update_rows(const vector<const vector<string>*>& replace_rows, const vector<const vector<string>*>& delete_rows)
{
  if (! replace_rows.empty()) {
    this->replace_rows(*dbh_, replace_rows);
  }
  if (! delete_rows.empty()) {
    this->delete_rows(*dbh_, delete_rows);
  }
  return true;
}

string
incline_driver_async_qtable::forwarder::do_get_extra_cond()
{
  return string();
}

void
incline_driver_async_qtable::forwarder::replace_rows(tmd::conn_t& dbh,
						     const vector<const vector<string>*>& rows) const
{
  string sql = replace_row_query_base_ + '(';
  for (vector<const vector<string>*>::const_iterator ri = rows.begin();
       ri != rows.end();
       ++ri) {
    for (vector<string>::const_iterator ci = (*ri)->begin();
	 ci != (*ri)->end();
	 ++ci) {
      sql.push_back('\'');
      sql += tmd::escape(dbh, *ci);
      sql += "',";
    }
    sql.erase(sql.size() - 1);
    sql += "),(";
  }
  sql.erase(sql.size() - 2);
  mgr_->log_sql(sql);
  tmd::execute(dbh, sql);
}

void
incline_driver_async_qtable::forwarder::delete_rows(tmd::conn_t& dbh,
						    const vector<const vector<string>*>& pk_rows) const
{
  vector<string> conds;
  for (vector<const vector<string>*>::const_iterator pi = pk_rows.begin();
       pi != pk_rows.end();
       ++pi) {
    conds.push_back("(" + _build_pk_cond(dbh, dest_pk_columns_, **pi) + ')');
  }
  string sql = delete_row_query_base_ + incline_util::join(" OR ", conds);
  mgr_->log_sql(sql);
  tmd::execute(dbh, sql);
}

string
incline_driver_async_qtable::forwarder::_build_pk_cond(tmd::conn_t& dbh,
						       const vector<string>&
						       colnames,
						       const vector<string>&
						       rows)
{
  assert(colnames.size() == rows.size());
  vector<string> cond;
  for (size_t i = 0; i < rows.size(); ++i) {
    cond.push_back(colnames[i] + "='" + tmd::escape(dbh, rows[i]) + '\'');
  }
  return incline_util::join(" AND ", cond);
}

void*
incline_driver_async_qtable::forwarder_mgr::run()
{
  vector<pthread_t> threads;

  { // create and start forwarders
    const vector<incline_def*>& defs = driver()->mgr()->defs();
    for (vector<incline_def*>::const_iterator di = defs.begin();
	 di != defs.end();
	 ++di) {
      const incline_def_async_qtable* def
	= dynamic_cast<const incline_def_async_qtable*>(*di);
      assert(def != NULL);
      threads.push_back(start_thread(do_create_forwarder(def)));
    }
  }

  // loop
  while (! threads.empty()) {
    pthread_join(threads.back(), NULL);
    threads.pop_back();
  }

  return NULL;
}

void
incline_driver_async_qtable::forwarder_mgr::log_sql(const string& sql)
{
  if (log_fd_ != -1) {
    struct iovec vec[2];
    vec[0].iov_base = const_cast<char*>(sql.c_str());
    vec[0].iov_len = sql.size();
    vec[1].iov_base = const_cast<char*>(";\n");
    vec[1].iov_len = 2;
    writev(log_fd_, vec, 2);
  }
}

incline_driver_async_qtable::forwarder*
incline_driver_async_qtable::forwarder_mgr::do_create_forwarder(const incline_def_async_qtable* def)
{
  tmd::conn_t* dbh = (*connect_)(src_host_.c_str(), src_port_);
  assert(dbh != NULL);
  return new forwarder(this, def, dbh, poll_interval_);
}
<|endoftext|>


<commit_before>// This utility will act as an aid in order to identify the position of the cameras and generates two files: and automatically generate
// 1- A ROS launch file which can be used to start all the ROS video streams
// 2-

#include <string>
#include <iostream>
#include <stdio.h>
#include <fstream>
#include <tinyxml.h>
#include <openni2/OpenNI.h>
#include "openni2_camera/openni2_driver.h"
#include "openni2_camera/openni2_device_manager.h"
#include "openni2_camera/openni2_exception.h"
#include <openni2_camera/openni2_device.h>
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

// OpenCV stuff
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
// OpenCV to ROS stuff
#include <cv_bridge/cv_bridge.h>

#include <boost/shared_ptr.hpp>
#include <boost/concept_check.hpp>

using namespace std;
using openni2_wrapper::OpenNI2DeviceManager;
using openni2_wrapper::OpenNI2DeviceInfo;
using openni2_wrapper::OpenNI2Exception;

openni2_wrapper::OpenNI2DeviceManager manager;

void newColorFrameCallback(sensor_msgs::ImagePtr image); // Get the images from the OpenNI2 stream

// Global flag stuff... the demon is inside hahahaha
int img_num = 0;
bool wait = false;
int max_wait = 10; // Maximum wait time in seconds
string device_id; // ID of the current camera
int data_skip = 0; // Data skip values

struct device_info {
  string camera_name;
  string serial;
  string vendor_id;
  string product_id;
  string location;

  string toString() const {
    ostringstream os;
    os << "Camera name: " << camera_name << "\n";
    os << "Vendor ID: " << vendor_id << "\n";
    os << "Product ID: " << product_id<< "\n";
    os << "Serial: " << serial << "\n";
    os << "Location: " << location << "\n";
    return os.str();
  }
};

vector <device_info> getCamerasInfo();
bool saveLaunchFile(const string& s, const vector <device_info> &cameras, const string &input_file);
void getDefaultParametersFromLaunchFile(const std::string &launch_file, TiXmlElement *launch_element);
bool saveUdevRules(const string &s, const vector <device_info> &cameras);
// Translates the URI of the device into a path: /dev/bus/usb/...
string getUsbLocation(const string &device_uri);
string getSerialNumber(const string &device_location);
void addArgumentTags(TiXmlElement& elem_add, const TiXmlElement& elem_source);

int main (int argc, char **argv) {
  if (argc < 3) {
    cerr << "Usage: " << argv[0] << " <launch_filename> <input_filename> [<data_skip>]\n";
    return -1;
  }

  if (argc >=3) {
    data_skip = atoi(argv[3]);
  }

  const std::string window_name = "image_show";
  cv::namedWindow(window_name, cv::WINDOW_AUTOSIZE ); // Create a window for display.

  vector<device_info> camera_names = getCamerasInfo(); // Save the camera names

  if (camera_names.size() > 0) {
    cout << "Detected " << camera_names.size() << " cameras. Info:\n";

    for (uint i = 0; i < camera_names.size(); i++) {
      cout << camera_names.at(i).toString();
    }

    cout << "Saving launch file.\n";

    if (saveLaunchFile(string(argv[1]), camera_names, string(argv[2]))) {
      cout << "Launch file saved successfully.\n";
    }
  }

  cv::destroyWindow("image_show");

  return 0;
}

// Uses tinyxml to generate a launch file with
bool saveLaunchFile(const string &s, const vector<device_info> &camera_info_vec, const string &input_file) {
  bool ret_val = true;
  // Create document
  TiXmlDocument doc;
  TiXmlDeclaration decl( "1.0", "", "" );
  doc.InsertEndChild( decl );

  // Create root launch node
  TiXmlElement launch_element("launch");

  getDefaultParametersFromLaunchFile(input_file, &launch_element);

  TiXmlElement *arg_skip_tag = new TiXmlElement("arg");
  arg_skip_tag->SetAttribute("name", "data_skip");
  arg_skip_tag->SetAttribute("default", data_skip);
  launch_element.LinkEndChild(arg_skip_tag);
  for (int i = 0; i < camera_info_vec.size(); i++) {
    const device_info &curr_cam = camera_info_vec.at(i);
    TiXmlElement *include_elem = new TiXmlElement("include");

    // File attribute: the default launch file of openni2
    include_elem->SetAttribute("file", "$(find openni2_launch)/launch/openni2.launch");

    // Tag argument 1: device_id
    TiXmlElement *arg_tag = new TiXmlElement("arg");
    arg_tag->SetAttribute("name", "device_id");
    arg_tag->SetAttribute("value", curr_cam.serial);
    include_elem->LinkEndChild(arg_tag);

    // Second tag --> argument name of the camera
    arg_tag = new TiXmlElement("arg");
    arg_tag->SetAttribute("name", "camera");
    arg_tag->SetAttribute("value", curr_cam.camera_name);
    include_elem->LinkEndChild(arg_tag);

    TiXmlElement *param_tag = new TiXmlElement("param");
    string s("/");
    s.append(curr_cam.camera_name);
    s.append("/driver/data_skip");
    param_tag->SetAttribute("name", s);
    param_tag->SetAttribute("value", "$(arg data_skip)");
    include_elem->LinkEndChild(param_tag);

    addArgumentTags(*include_elem, launch_element);

    launch_element.LinkEndChild(include_elem);
  }

  doc.InsertEndChild(launch_element);
  doc.SaveFile(s);

  return ret_val;
}

vector<device_info> getCamerasInfo()
{
  vector<device_info> ret_val;

  // Get the connected OpenNI2 devices
  boost::shared_ptr<std::vector<openni2_wrapper::OpenNI2DeviceInfo> > device_infos = manager.getConnectedDeviceInfos();
  std::cout << "Found " << device_infos->size() << " devices:" << std::endl;

  // Iterate over the devices, asking the user for labels and generating the proper include tag
  for (size_t i = 0; i < device_infos->size(); ++i)
  {
    openni2_wrapper::OpenNI2DeviceInfo &info = device_infos->at(i);
    device_info camera_info;

    ostringstream os, os2;
    os << "camera_" << i;
    std::string camera_label(os.str());

    os2 << "#" << i + 1;
    device_id = os2.str();

    cout << "Showing the RGB image associated with camera " << device_id << endl;

    try {
      img_num = 0;
      wait = true;
      boost::shared_ptr<openni2_wrapper::OpenNI2Device> device = manager.getDevice(info.uri_);

      if (device->hasColorSensor()) {
	device->setColorFrameCallback(newColorFrameCallback);
	device->startColorStream();
      }

      int cont = 0;
      // Wait for the image to be shown and a key pressed
      while (wait && cont < max_wait) {
	cont++;
	sleep(1);
      }
      if (wait) {
	// No image has been found
	cerr << "Warning: Could not retrieve image from camera " << device_id << ". Setting the label to camera_" << i + 1 << endl;
	wait = false;
      } else {
	cout << "Please enter the label for camera " << device_id << endl;
	getline(cin, camera_label);
      }

      // Save the camera info
      camera_info.camera_name = camera_label;
      std::ostringstream product_id, vendor_id;
      product_id << std::hex << setfill('0') << setw(4) << info.product_id_;
      camera_info.product_id = product_id.str();
      vendor_id << std::hex << setfill('0') << setw(4) << std::hex << info.vendor_id_;
      camera_info.vendor_id = vendor_id.str();
      camera_info.location = getUsbLocation(info.uri_);
      camera_info.serial = manager.getSerial(info.uri_);
      ret_val.push_back(camera_info);
    } catch (exception &e) {
      cerr << "Exception thrown while managing camera " << device_id << ". Content: " << e.what() << endl;
    }
  }

  return ret_val;
}

// Get the accepted arguments and their default values from another launch file (typically openni2_launch/launch/openni2.launch)
void getDefaultParametersFromLaunchFile(const std::string &launch_file, TiXmlElement *launch_element) {
  // Load the file where the default parameters will be stored
  TiXmlDocument doc(launch_file);
  doc.LoadFile();
  TiXmlElement *root = doc.RootElement();

  if (!root) {
    cerr << "Could not get the launch file.\n";
    return;
  }

  // Iterate over the children and copy the argument data
  TiXmlNode *it = root->FirstChild();
  while (it) {
    if (it->ValueStr() == "arg" && it->ToElement()) {
      string name(it->ToElement()->Attribute("name"));
      // Discard undesired tags
      if (name != "camera" && name != "rgb_frame_id" && name != "device_id" && name != "rgb_frame_id" && name != "depth_frame_id" && name != "depth_camera_info_url" &&
	name != "rgb_camera_info_url")
      {
	TiXmlElement *node = new TiXmlElement("arg");

	node->SetAttribute("name", it->ToElement()->Attribute("name"));
	node->SetAttribute("default", it->ToElement()->Attribute("default"));

	if (it->ToElement()->Attribute("if")) {
	  node->SetAttribute("if", it->ToElement()->Attribute("if"));
	} else if (it->ToElement()->Attribute("unless")) {
	  node->SetAttribute("unless", it->ToElement()->Attribute("unless"));
	}

	launch_element->LinkEndChild(node);
      }
    }
    // Next getXmlCameraElement
    it = root->IterateChildren(it);
  }
}

void newColorFrameCallback(sensor_msgs::ImagePtr image)
{
  if (img_num == 0 && wait) {
    img_num++;
    cv_bridge::CvImagePtr img_cv = cv_bridge::toCvCopy(image, "bgr8");

    ostringstream os;
    os << "Camera " << device_id;

    cv::imshow("image_show", img_cv.get()->image );                   // Show our image inside it.
    cv::updateWindow ("image_show");
    cv::startWindowThread();
    cv::waitKey(0); // Wait for a key to be pressed in the window
    wait = false;
  }
}

string getUsbLocation(const string& device_uri)
{
  string ret = "/dev/bus/usb/";
  int i;

  string bus_device = device_uri.substr(device_uri.find('@') + 1, device_uri.size());
  istringstream bus_is (bus_device.substr(0, bus_device.find('/')));
  bus_is >> i;
  ostringstream bus_id;
  bus_id << setfill('0') << setw(3) << i;

  istringstream is (bus_device.substr(bus_device.find('/') + 1, bus_device.size()));

  is >> i;
  ostringstream device_id;
  device_id << setfill('0') << setw(3) << i;

  ret.append(bus_id.str());
  ret.append("/");
  ret.append(device_id.str());

  return ret;
}

void addArgumentTags(TiXmlElement& elem_add, const TiXmlElement& elem_source)
{
  // Iterate over the children and copy the argument data
  const TiXmlNode *it = elem_source.FirstChild();
  while (it) {
    if (it->ValueStr() == "arg" && it->ToElement() && static_cast<string>(it->ToElement()->Attribute("name")) != "device_id") {
      TiXmlElement *node = new TiXmlElement("arg");

      node->SetAttribute("name", it->ToElement()->Attribute("name"));
      ostringstream os;
      os << "$(arg " << it->ToElement()->Attribute("name") << ")";
      node->SetAttribute("value", os.str());
      elem_add.LinkEndChild(node);
    }
    // Next getXmlCameraElement
    it = elem_source.IterateChildren(it);
  }
}

<commit_msg>Added a subtle documentation on data skip<commit_after>// This utility will act as an aid in order to identify the position of the cameras and generates two files: and automatically generate
// 1- A ROS launch file which can be used to start all the ROS video streams
// 2-

#include <string>
#include <iostream>
#include <stdio.h>
#include <fstream>
#include <tinyxml.h>
#include <openni2/OpenNI.h>
#include "openni2_camera/openni2_driver.h"
#include "openni2_camera/openni2_device_manager.h"
#include "openni2_camera/openni2_exception.h"
#include <openni2_camera/openni2_device.h>
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

// OpenCV stuff
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
// OpenCV to ROS stuff
#include <cv_bridge/cv_bridge.h>

#include <boost/shared_ptr.hpp>
#include <boost/concept_check.hpp>

using namespace std;
using openni2_wrapper::OpenNI2DeviceManager;
using openni2_wrapper::OpenNI2DeviceInfo;
using openni2_wrapper::OpenNI2Exception;

openni2_wrapper::OpenNI2DeviceManager manager;

void newColorFrameCallback(sensor_msgs::ImagePtr image); // Get the images from the OpenNI2 stream

// Global flag stuff... the demon is inside hahahaha
int img_num = 0;
bool wait = false;
int max_wait = 10; // Maximum wait time in seconds
string device_id; // ID of the current camera
int data_skip = 0; // Data skip values

struct device_info {
  string camera_name;
  string serial;
  string vendor_id;
  string product_id;
  string location;

  string toString() const {
    ostringstream os;
    os << "Camera name: " << camera_name << "\n";
    os << "Vendor ID: " << vendor_id << "\n";
    os << "Product ID: " << product_id<< "\n";
    os << "Serial: " << serial << "\n";
    os << "Location: " << location << "\n";
    return os.str();
  }
};

vector <device_info> getCamerasInfo();
bool saveLaunchFile(const string& s, const vector <device_info> &cameras, const string &input_file);
void getDefaultParametersFromLaunchFile(const std::string &launch_file, TiXmlElement *launch_element);
bool saveUdevRules(const string &s, const vector <device_info> &cameras);
// Translates the URI of the device into a path: /dev/bus/usb/...
string getUsbLocation(const string &device_uri);
string getSerialNumber(const string &device_location);
void addArgumentTags(TiXmlElement& elem_add, const TiXmlElement& elem_source);

int main (int argc, char **argv) {
  if (argc < 3) {
    cerr << "Usage: " << argv[0] << " <launch_filename> <input_filename> [<data_skip>]\n";
    cerr << "Data skip means the number of frames necessary in order to publish one. I.e. data skip = 3 will publish 1 image of each 3 received\n";
    return -1;
  }

  if (argc >=3) {
    data_skip = atoi(argv[3]);
  }

  const std::string window_name = "image_show";
  cv::namedWindow(window_name, cv::WINDOW_AUTOSIZE ); // Create a window for display.

  vector<device_info> camera_names = getCamerasInfo(); // Save the camera names

  if (camera_names.size() > 0) {
    cout << "Detected " << camera_names.size() << " cameras. Info:\n";

    for (uint i = 0; i < camera_names.size(); i++) {
      cout << camera_names.at(i).toString();
    }

    cout << "Saving launch file.\n";

    if (saveLaunchFile(string(argv[1]), camera_names, string(argv[2]))) {
      cout << "Launch file saved successfully.\n";
    }
  }

  cv::destroyWindow("image_show");

  return 0;
}

// Uses tinyxml to generate a launch file with
bool saveLaunchFile(const string &s, const vector<device_info> &camera_info_vec, const string &input_file) {
  bool ret_val = true;
  // Create document
  TiXmlDocument doc;
  TiXmlDeclaration decl( "1.0", "", "" );
  doc.InsertEndChild( decl );

  // Create root launch node
  TiXmlElement launch_element("launch");

  getDefaultParametersFromLaunchFile(input_file, &launch_element);

  TiXmlElement *arg_skip_tag = new TiXmlElement("arg");
  arg_skip_tag->SetAttribute("name", "data_skip");
  arg_skip_tag->SetAttribute("default", data_skip);
  launch_element.LinkEndChild(arg_skip_tag);
  for (int i = 0; i < camera_info_vec.size(); i++) {
    const device_info &curr_cam = camera_info_vec.at(i);
    TiXmlElement *include_elem = new TiXmlElement("include");

    // File attribute: the default launch file of openni2
    include_elem->SetAttribute("file", "$(find openni2_launch)/launch/openni2.launch");

    // Tag argument 1: device_id
    TiXmlElement *arg_tag = new TiXmlElement("arg");
    arg_tag->SetAttribute("name", "device_id");
    arg_tag->SetAttribute("value", curr_cam.serial);
    include_elem->LinkEndChild(arg_tag);

    // Second tag --> argument name of the camera
    arg_tag = new TiXmlElement("arg");
    arg_tag->SetAttribute("name", "camera");
    arg_tag->SetAttribute("value", curr_cam.camera_name);
    include_elem->LinkEndChild(arg_tag);

    TiXmlElement *param_tag = new TiXmlElement("param");
    string s("/");
    s.append(curr_cam.camera_name);
    s.append("/driver/data_skip");
    param_tag->SetAttribute("name", s);
    param_tag->SetAttribute("value", "$(arg data_skip)");
    include_elem->LinkEndChild(param_tag);

    addArgumentTags(*include_elem, launch_element);

    launch_element.LinkEndChild(include_elem);
  }

  doc.InsertEndChild(launch_element);
  doc.SaveFile(s);

  return ret_val;
}

vector<device_info> getCamerasInfo()
{
  vector<device_info> ret_val;

  // Get the connected OpenNI2 devices
  boost::shared_ptr<std::vector<openni2_wrapper::OpenNI2DeviceInfo> > device_infos = manager.getConnectedDeviceInfos();
  std::cout << "Found " << device_infos->size() << " devices:" << std::endl;

  // Iterate over the devices, asking the user for labels and generating the proper include tag
  for (size_t i = 0; i < device_infos->size(); ++i)
  {
    openni2_wrapper::OpenNI2DeviceInfo &info = device_infos->at(i);
    device_info camera_info;

    ostringstream os, os2;
    os << "camera_" << i;
    std::string camera_label(os.str());

    os2 << "#" << i + 1;
    device_id = os2.str();

    cout << "Showing the RGB image associated with camera " << device_id << endl;

    try {
      img_num = 0;
      wait = true;
      boost::shared_ptr<openni2_wrapper::OpenNI2Device> device = manager.getDevice(info.uri_);

      if (device->hasColorSensor()) {
	device->setColorFrameCallback(newColorFrameCallback);
	device->startColorStream();
      }

      int cont = 0;
      // Wait for the image to be shown and a key pressed
      while (wait && cont < max_wait) {
	cont++;
	sleep(1);
      }
      if (wait) {
	// No image has been found
	cerr << "Warning: Could not retrieve image from camera " << device_id << ". Setting the label to camera_" << i + 1 << endl;
	wait = false;
      } else {
	cout << "Please enter the label for camera " << device_id << endl;
	getline(cin, camera_label);
      }

      // Save the camera info
      camera_info.camera_name = camera_label;
      std::ostringstream product_id, vendor_id;
      product_id << std::hex << setfill('0') << setw(4) << info.product_id_;
      camera_info.product_id = product_id.str();
      vendor_id << std::hex << setfill('0') << setw(4) << std::hex << info.vendor_id_;
      camera_info.vendor_id = vendor_id.str();
      camera_info.location = getUsbLocation(info.uri_);
      camera_info.serial = manager.getSerial(info.uri_);
      ret_val.push_back(camera_info);
    } catch (exception &e) {
      cerr << "Exception thrown while managing camera " << device_id << ". Content: " << e.what() << endl;
    }
  }

  return ret_val;
}

// Get the accepted arguments and their default values from another launch file (typically openni2_launch/launch/openni2.launch)
void getDefaultParametersFromLaunchFile(const std::string &launch_file, TiXmlElement *launch_element) {
  // Load the file where the default parameters will be stored
  TiXmlDocument doc(launch_file);
  doc.LoadFile();
  TiXmlElement *root = doc.RootElement();

  if (!root) {
    cerr << "Could not get the launch file.\n";
    return;
  }

  // Iterate over the children and copy the argument data
  TiXmlNode *it = root->FirstChild();
  while (it) {
    if (it->ValueStr() == "arg" && it->ToElement()) {
      string name(it->ToElement()->Attribute("name"));
      // Discard undesired tags
      if (name != "camera" && name != "rgb_frame_id" && name != "device_id" && name != "rgb_frame_id" && name != "depth_frame_id" && name != "depth_camera_info_url" &&
	name != "rgb_camera_info_url")
      {
	TiXmlElement *node = new TiXmlElement("arg");

	node->SetAttribute("name", it->ToElement()->Attribute("name"));
	node->SetAttribute("default", it->ToElement()->Attribute("default"));

	if (it->ToElement()->Attribute("if")) {
	  node->SetAttribute("if", it->ToElement()->Attribute("if"));
	} else if (it->ToElement()->Attribute("unless")) {
	  node->SetAttribute("unless", it->ToElement()->Attribute("unless"));
	}

	launch_element->LinkEndChild(node);
      }
    }
    // Next getXmlCameraElement
    it = root->IterateChildren(it);
  }
}

void newColorFrameCallback(sensor_msgs::ImagePtr image)
{
  if (img_num == 0 && wait) {
    img_num++;
    cv_bridge::CvImagePtr img_cv = cv_bridge::toCvCopy(image, "bgr8");

    ostringstream os;
    os << "Camera " << device_id;

    cv::imshow("image_show", img_cv.get()->image );                   // Show our image inside it.
    cv::updateWindow ("image_show");
    cv::startWindowThread();
    cv::waitKey(0); // Wait for a key to be pressed in the window
    wait = false;
  }
}

string getUsbLocation(const string& device_uri)
{
  string ret = "/dev/bus/usb/";
  int i;

  string bus_device = device_uri.substr(device_uri.find('@') + 1, device_uri.size());
  istringstream bus_is (bus_device.substr(0, bus_device.find('/')));
  bus_is >> i;
  ostringstream bus_id;
  bus_id << setfill('0') << setw(3) << i;

  istringstream is (bus_device.substr(bus_device.find('/') + 1, bus_device.size()));

  is >> i;
  ostringstream device_id;
  device_id << setfill('0') << setw(3) << i;

  ret.append(bus_id.str());
  ret.append("/");
  ret.append(device_id.str());

  return ret;
}

void addArgumentTags(TiXmlElement& elem_add, const TiXmlElement& elem_source)
{
  // Iterate over the children and copy the argument data
  const TiXmlNode *it = elem_source.FirstChild();
  while (it) {
    if (it->ValueStr() == "arg" && it->ToElement() && static_cast<string>(it->ToElement()->Attribute("name")) != "device_id") {
      TiXmlElement *node = new TiXmlElement("arg");

      node->SetAttribute("name", it->ToElement()->Attribute("name"));
      ostringstream os;
      os << "$(arg " << it->ToElement()->Attribute("name") << ")";
      node->SetAttribute("value", os.str());
      elem_add.LinkEndChild(node);
    }
    // Next getXmlCameraElement
    it = elem_source.IterateChildren(it);
  }
}

<|endoftext|>


<commit_before>/*
 * Copyright © 2010 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include "ast.h"
#include "glsl_types.h"
#include "ir.h"

void
ast_array_specifier::print(void) const
{
   foreach_list_typed (ast_node, array_dimension, link, &this->array_dimensions) {
      printf("[ ");
      if (((ast_expression*)array_dimension)->oper != ast_unsized_array_dim)
         array_dimension->print();
      printf("] ");
   }
}

/**
 * If \c ir is a reference to an array for which we are tracking the max array
 * element accessed, track that the given element has been accessed.
 * Otherwise do nothing.
 *
 * This function also checks whether the array is a built-in array whose
 * maximum size is too small to accommodate the given index, and if so uses
 * loc and state to report the error.
 */
static void
update_max_array_access(ir_rvalue *ir, int idx, YYLTYPE *loc,
                        struct _mesa_glsl_parse_state *state)
{
   if (ir_dereference_variable *deref_var = ir->as_dereference_variable()) {
      ir_variable *var = deref_var->var;
      if (idx > (int)var->data.max_array_access) {
         var->data.max_array_access = idx;

         /* Check whether this access will, as a side effect, implicitly cause
          * the size of a built-in array to be too large.
          */
         check_builtin_array_max_size(var->name, idx+1, *loc, state);
      }
   } else if (ir_dereference_record *deref_record =
              ir->as_dereference_record()) {
      /* There are two possibilities we need to consider:
       *
       * - Accessing an element of an array that is a member of a named
       *   interface block (e.g. ifc.foo[i])
       *
       * - Accessing an element of an array that is a member of a named
       *   interface block array (e.g. ifc[j].foo[i]).
       */
      ir_dereference_variable *deref_var =
         deref_record->record->as_dereference_variable();
      if (deref_var == NULL) {
         if (ir_dereference_array *deref_array =
             deref_record->record->as_dereference_array()) {
            deref_var = deref_array->array->as_dereference_variable();
         }
      }

      if (deref_var != NULL) {
         if (deref_var->var->is_interface_instance()) {
            unsigned field_index =
               deref_record->record->type->field_index(deref_record->field);
            assert(field_index < deref_var->var->get_interface_type()->length);

            unsigned *const max_ifc_array_access =
               deref_var->var->get_max_ifc_array_access();

            assert(max_ifc_array_access != NULL);

            if (idx > (int)max_ifc_array_access[field_index]) {
               max_ifc_array_access[field_index] = idx;

               /* Check whether this access will, as a side effect, implicitly
                * cause the size of a built-in array to be too large.
                */
               check_builtin_array_max_size(deref_record->field, idx+1, *loc,
                                            state);
            }
         }
      }
   }
}

static int
get_implicit_array_size(struct _mesa_glsl_parse_state *state,
                        ir_rvalue *array)
{
   ir_variable *var = array->variable_referenced();

   /* Inputs in control shader are implicitly sized
    * to the maximum patch size.
    */
   if (state->stage == MESA_SHADER_TESS_CTRL &&
       var->data.mode == ir_var_shader_in) {
      return state->Const.MaxPatchVertices;
   }

   /* Non-patch inputs in evaluation shader are implicitly sized
    * to the maximum patch size.
    */
   if (state->stage == MESA_SHADER_TESS_EVAL &&
       var->data.mode == ir_var_shader_in &&
       !var->data.patch) {
      return state->Const.MaxPatchVertices;
   }

   return 0;
}

ir_rvalue *
_mesa_ast_array_index_to_hir(void *mem_ctx,
			     struct _mesa_glsl_parse_state *state,
			     ir_rvalue *array, ir_rvalue *idx,
			     YYLTYPE &loc, YYLTYPE &idx_loc)
{
   if (!array->type->is_error()
       && !array->type->is_array()
       && !array->type->is_matrix()
       && !array->type->is_vector()) {
      _mesa_glsl_error(& idx_loc, state,
		       "cannot dereference non-array / non-matrix / "
		       "non-vector");
   }

   if (!idx->type->is_error()) {
      if (!idx->type->is_integer()) {
	 _mesa_glsl_error(& idx_loc, state, "array index must be integer type");
      } else if (!idx->type->is_scalar()) {
	 _mesa_glsl_error(& idx_loc, state, "array index must be scalar");
      }
   }

   /* If the array index is a constant expression and the array has a
    * declared size, ensure that the access is in-bounds.  If the array
    * index is not a constant expression, ensure that the array has a
    * declared size.
    */
   ir_constant *const const_index = idx->constant_expression_value();
   if (const_index != NULL && idx->type->is_integer()) {
      const int idx = const_index->value.i[0];
      const char *type_name = "error";
      unsigned bound = 0;

      /* From page 24 (page 30 of the PDF) of the GLSL 1.50 spec:
       *
       *    "It is illegal to declare an array with a size, and then
       *    later (in the same shader) index the same array with an
       *    integral constant expression greater than or equal to the
       *    declared size. It is also illegal to index an array with a
       *    negative constant expression."
       */
      if (array->type->is_matrix()) {
	 if (array->type->row_type()->vector_elements <= idx) {
	    type_name = "matrix";
	    bound = array->type->row_type()->vector_elements;
	 }
      } else if (array->type->is_vector()) {
	 if (array->type->vector_elements <= idx) {
	    type_name = "vector";
	    bound = array->type->vector_elements;
	 }
      } else {
	 /* glsl_type::array_size() returns -1 for non-array types.  This means
	  * that we don't need to verify that the type is an array before
	  * doing the bounds checking.
	  */
	 if ((array->type->array_size() > 0)
	     && (array->type->array_size() <= idx)) {
	    type_name = "array";
	    bound = array->type->array_size();
	 }
      }

      if (bound > 0) {
	 _mesa_glsl_error(& loc, state, "%s index must be < %u",
			  type_name, bound);
      } else if (idx < 0) {
	 _mesa_glsl_error(& loc, state, "%s index must be >= 0",
			  type_name);
      }

      if (array->type->is_array())
         update_max_array_access(array, idx, &loc, state);
   } else if (const_index == NULL && array->type->is_array()) {
      if (array->type->is_unsized_array()) {
         int implicit_size = get_implicit_array_size(state, array);
         if (implicit_size) {
            ir_variable *v = array->whole_variable_referenced();
            if (v != NULL)
               v->data.max_array_access = implicit_size - 1;
         }
         else if (state->stage == MESA_SHADER_TESS_CTRL &&
                  array->variable_referenced()->data.mode == ir_var_shader_out &&
                  !array->variable_referenced()->data.patch) {
            /* Tessellation control shader output non-patch arrays are
             * initially unsized. Despite that, they are allowed to be
             * indexed with a non-constant expression (typically
             * "gl_InvocationID"). The array size will be determined
             * by the linker.
             */
         }
         else if (array->variable_referenced()->data.mode !=
                  ir_var_shader_storage) {
            _mesa_glsl_error(&loc, state, "unsized array index must be constant");
         }
      } else if (array->type->fields.array->is_interface()
                 && (array->variable_referenced()->data.mode == ir_var_uniform ||
                     array->variable_referenced()->data.mode == ir_var_shader_storage)
                 && !state->is_version(400, 0) && !state->ARB_gpu_shader5_enable) {
	 /* Page 50 in section 4.3.9 of the OpenGL ES 3.10 spec says:
	  *
	  *     "All indices used to index a uniform or shader storage block
	  *     array must be constant integral expressions."
	  */
	 _mesa_glsl_error(&loc, state, "%s block array index must be constant",
                          array->variable_referenced()->data.mode
                          == ir_var_uniform ? "uniform" : "shader storage");
      } else {
	 /* whole_variable_referenced can return NULL if the array is a
	  * member of a structure.  In this case it is safe to not update
	  * the max_array_access field because it is never used for fields
	  * of structures.
	  */
	 ir_variable *v = array->whole_variable_referenced();
	 if (v != NULL)
	    v->data.max_array_access = array->type->array_size() - 1;
      }

      /* From page 23 (29 of the PDF) of the GLSL 1.30 spec:
       *
       *    "Samplers aggregated into arrays within a shader (using square
       *    brackets [ ]) can only be indexed with integral constant
       *    expressions [...]."
       *
       * This restriction was added in GLSL 1.30.  Shaders using earlier
       * version of the language should not be rejected by the compiler
       * front-end for using this construct.  This allows useful things such
       * as using a loop counter as the index to an array of samplers.  If the
       * loop in unrolled, the code should compile correctly.  Instead, emit a
       * warning.
       *
       * In GLSL 4.00 / ARB_gpu_shader5, this requirement is relaxed again to allow
       * indexing with dynamically uniform expressions. Note that these are not
       * required to be uniforms or expressions based on them, but merely that the
       * values must not diverge between shader invocations run together. If the
       * values *do* diverge, then the behavior of the operation requiring a
       * dynamically uniform expression is undefined.
       */
      if (array->type->without_array()->is_sampler()) {
         if (!state->is_version(400, 0) && !state->ARB_gpu_shader5_enable) {
            if (state->is_version(130, 300))
               _mesa_glsl_error(&loc, state,
                                "sampler arrays indexed with non-constant "
                                "expressions are forbidden in GLSL %s "
                                "and later",
                                state->es_shader ? "ES 3.00" : "1.30");
            else if (state->es_shader)
               _mesa_glsl_warning(&loc, state,
                                  "sampler arrays indexed with non-constant "
                                  "expressions will be forbidden in GLSL "
                                  "3.00 and later");
            else
               _mesa_glsl_warning(&loc, state,
                                  "sampler arrays indexed with non-constant "
                                  "expressions will be forbidden in GLSL "
                                  "1.30 and later");
         }
      }

      /* From page 27 of the GLSL ES 3.1 specification:
       *
       * "When aggregated into arrays within a shader, images can only be
       *  indexed with a constant integral expression."
       *
       * On the other hand the desktop GL specification extension allows
       * non-constant indexing of image arrays, but behavior is left undefined
       * in cases where the indexing expression is not dynamically uniform.
       */
      if (state->es_shader && array->type->without_array()->is_image()) {
         _mesa_glsl_error(&loc, state,
                          "image arrays indexed with non-constant "
                          "expressions are forbidden in GLSL ES.");
      }
   }

   /* After performing all of the error checking, generate the IR for the
    * expression.
    */
   if (array->type->is_array()
       || array->type->is_matrix()) {
      return new(mem_ctx) ir_dereference_array(array, idx);
   } else if (array->type->is_vector()) {
      return new(mem_ctx) ir_expression(ir_binop_vector_extract, array, idx);
   } else if (array->type->is_error()) {
      return array;
   } else {
      ir_rvalue *result = new(mem_ctx) ir_dereference_array(array, idx);
      result->type = glsl_type::error_type;

      return result;
   }
}
<commit_msg>glsl: add AoA support for an inteface with unsized array members<commit_after>/*
 * Copyright © 2010 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include "ast.h"
#include "glsl_types.h"
#include "ir.h"

void
ast_array_specifier::print(void) const
{
   foreach_list_typed (ast_node, array_dimension, link, &this->array_dimensions) {
      printf("[ ");
      if (((ast_expression*)array_dimension)->oper != ast_unsized_array_dim)
         array_dimension->print();
      printf("] ");
   }
}

/**
 * If \c ir is a reference to an array for which we are tracking the max array
 * element accessed, track that the given element has been accessed.
 * Otherwise do nothing.
 *
 * This function also checks whether the array is a built-in array whose
 * maximum size is too small to accommodate the given index, and if so uses
 * loc and state to report the error.
 */
static void
update_max_array_access(ir_rvalue *ir, int idx, YYLTYPE *loc,
                        struct _mesa_glsl_parse_state *state)
{
   if (ir_dereference_variable *deref_var = ir->as_dereference_variable()) {
      ir_variable *var = deref_var->var;
      if (idx > (int)var->data.max_array_access) {
         var->data.max_array_access = idx;

         /* Check whether this access will, as a side effect, implicitly cause
          * the size of a built-in array to be too large.
          */
         check_builtin_array_max_size(var->name, idx+1, *loc, state);
      }
   } else if (ir_dereference_record *deref_record =
              ir->as_dereference_record()) {
      /* There are three possibilities we need to consider:
       *
       * - Accessing an element of an array that is a member of a named
       *   interface block (e.g. ifc.foo[i])
       *
       * - Accessing an element of an array that is a member of a named
       *   interface block array (e.g. ifc[j].foo[i]).
       *
       * - Accessing an element of an array that is a member of a named
       *   interface block array of arrays (e.g. ifc[j][k].foo[i]).
       */
      ir_dereference_variable *deref_var =
         deref_record->record->as_dereference_variable();
      if (deref_var == NULL) {
         ir_dereference_array *deref_array =
            deref_record->record->as_dereference_array();
         ir_dereference_array *deref_array_prev = NULL;
         while (deref_array != NULL) {
            deref_array_prev = deref_array;
            deref_array = deref_array->array->as_dereference_array();
         }
         if (deref_array_prev != NULL)
            deref_var = deref_array_prev->array->as_dereference_variable();
      }

      if (deref_var != NULL) {
         if (deref_var->var->is_interface_instance()) {
            unsigned field_index =
               deref_record->record->type->field_index(deref_record->field);
            assert(field_index < deref_var->var->get_interface_type()->length);

            unsigned *const max_ifc_array_access =
               deref_var->var->get_max_ifc_array_access();

            assert(max_ifc_array_access != NULL);

            if (idx > (int)max_ifc_array_access[field_index]) {
               max_ifc_array_access[field_index] = idx;

               /* Check whether this access will, as a side effect, implicitly
                * cause the size of a built-in array to be too large.
                */
               check_builtin_array_max_size(deref_record->field, idx+1, *loc,
                                            state);
            }
         }
      }
   }
}

static int
get_implicit_array_size(struct _mesa_glsl_parse_state *state,
                        ir_rvalue *array)
{
   ir_variable *var = array->variable_referenced();

   /* Inputs in control shader are implicitly sized
    * to the maximum patch size.
    */
   if (state->stage == MESA_SHADER_TESS_CTRL &&
       var->data.mode == ir_var_shader_in) {
      return state->Const.MaxPatchVertices;
   }

   /* Non-patch inputs in evaluation shader are implicitly sized
    * to the maximum patch size.
    */
   if (state->stage == MESA_SHADER_TESS_EVAL &&
       var->data.mode == ir_var_shader_in &&
       !var->data.patch) {
      return state->Const.MaxPatchVertices;
   }

   return 0;
}

ir_rvalue *
_mesa_ast_array_index_to_hir(void *mem_ctx,
			     struct _mesa_glsl_parse_state *state,
			     ir_rvalue *array, ir_rvalue *idx,
			     YYLTYPE &loc, YYLTYPE &idx_loc)
{
   if (!array->type->is_error()
       && !array->type->is_array()
       && !array->type->is_matrix()
       && !array->type->is_vector()) {
      _mesa_glsl_error(& idx_loc, state,
		       "cannot dereference non-array / non-matrix / "
		       "non-vector");
   }

   if (!idx->type->is_error()) {
      if (!idx->type->is_integer()) {
	 _mesa_glsl_error(& idx_loc, state, "array index must be integer type");
      } else if (!idx->type->is_scalar()) {
	 _mesa_glsl_error(& idx_loc, state, "array index must be scalar");
      }
   }

   /* If the array index is a constant expression and the array has a
    * declared size, ensure that the access is in-bounds.  If the array
    * index is not a constant expression, ensure that the array has a
    * declared size.
    */
   ir_constant *const const_index = idx->constant_expression_value();
   if (const_index != NULL && idx->type->is_integer()) {
      const int idx = const_index->value.i[0];
      const char *type_name = "error";
      unsigned bound = 0;

      /* From page 24 (page 30 of the PDF) of the GLSL 1.50 spec:
       *
       *    "It is illegal to declare an array with a size, and then
       *    later (in the same shader) index the same array with an
       *    integral constant expression greater than or equal to the
       *    declared size. It is also illegal to index an array with a
       *    negative constant expression."
       */
      if (array->type->is_matrix()) {
	 if (array->type->row_type()->vector_elements <= idx) {
	    type_name = "matrix";
	    bound = array->type->row_type()->vector_elements;
	 }
      } else if (array->type->is_vector()) {
	 if (array->type->vector_elements <= idx) {
	    type_name = "vector";
	    bound = array->type->vector_elements;
	 }
      } else {
	 /* glsl_type::array_size() returns -1 for non-array types.  This means
	  * that we don't need to verify that the type is an array before
	  * doing the bounds checking.
	  */
	 if ((array->type->array_size() > 0)
	     && (array->type->array_size() <= idx)) {
	    type_name = "array";
	    bound = array->type->array_size();
	 }
      }

      if (bound > 0) {
	 _mesa_glsl_error(& loc, state, "%s index must be < %u",
			  type_name, bound);
      } else if (idx < 0) {
	 _mesa_glsl_error(& loc, state, "%s index must be >= 0",
			  type_name);
      }

      if (array->type->is_array())
         update_max_array_access(array, idx, &loc, state);
   } else if (const_index == NULL && array->type->is_array()) {
      if (array->type->is_unsized_array()) {
         int implicit_size = get_implicit_array_size(state, array);
         if (implicit_size) {
            ir_variable *v = array->whole_variable_referenced();
            if (v != NULL)
               v->data.max_array_access = implicit_size - 1;
         }
         else if (state->stage == MESA_SHADER_TESS_CTRL &&
                  array->variable_referenced()->data.mode == ir_var_shader_out &&
                  !array->variable_referenced()->data.patch) {
            /* Tessellation control shader output non-patch arrays are
             * initially unsized. Despite that, they are allowed to be
             * indexed with a non-constant expression (typically
             * "gl_InvocationID"). The array size will be determined
             * by the linker.
             */
         }
         else if (array->variable_referenced()->data.mode !=
                  ir_var_shader_storage) {
            _mesa_glsl_error(&loc, state, "unsized array index must be constant");
         }
      } else if (array->type->fields.array->is_interface()
                 && (array->variable_referenced()->data.mode == ir_var_uniform ||
                     array->variable_referenced()->data.mode == ir_var_shader_storage)
                 && !state->is_version(400, 0) && !state->ARB_gpu_shader5_enable) {
	 /* Page 50 in section 4.3.9 of the OpenGL ES 3.10 spec says:
	  *
	  *     "All indices used to index a uniform or shader storage block
	  *     array must be constant integral expressions."
	  */
	 _mesa_glsl_error(&loc, state, "%s block array index must be constant",
                          array->variable_referenced()->data.mode
                          == ir_var_uniform ? "uniform" : "shader storage");
      } else {
	 /* whole_variable_referenced can return NULL if the array is a
	  * member of a structure.  In this case it is safe to not update
	  * the max_array_access field because it is never used for fields
	  * of structures.
	  */
	 ir_variable *v = array->whole_variable_referenced();
	 if (v != NULL)
	    v->data.max_array_access = array->type->array_size() - 1;
      }

      /* From page 23 (29 of the PDF) of the GLSL 1.30 spec:
       *
       *    "Samplers aggregated into arrays within a shader (using square
       *    brackets [ ]) can only be indexed with integral constant
       *    expressions [...]."
       *
       * This restriction was added in GLSL 1.30.  Shaders using earlier
       * version of the language should not be rejected by the compiler
       * front-end for using this construct.  This allows useful things such
       * as using a loop counter as the index to an array of samplers.  If the
       * loop in unrolled, the code should compile correctly.  Instead, emit a
       * warning.
       *
       * In GLSL 4.00 / ARB_gpu_shader5, this requirement is relaxed again to allow
       * indexing with dynamically uniform expressions. Note that these are not
       * required to be uniforms or expressions based on them, but merely that the
       * values must not diverge between shader invocations run together. If the
       * values *do* diverge, then the behavior of the operation requiring a
       * dynamically uniform expression is undefined.
       */
      if (array->type->without_array()->is_sampler()) {
         if (!state->is_version(400, 0) && !state->ARB_gpu_shader5_enable) {
            if (state->is_version(130, 300))
               _mesa_glsl_error(&loc, state,
                                "sampler arrays indexed with non-constant "
                                "expressions are forbidden in GLSL %s "
                                "and later",
                                state->es_shader ? "ES 3.00" : "1.30");
            else if (state->es_shader)
               _mesa_glsl_warning(&loc, state,
                                  "sampler arrays indexed with non-constant "
                                  "expressions will be forbidden in GLSL "
                                  "3.00 and later");
            else
               _mesa_glsl_warning(&loc, state,
                                  "sampler arrays indexed with non-constant "
                                  "expressions will be forbidden in GLSL "
                                  "1.30 and later");
         }
      }

      /* From page 27 of the GLSL ES 3.1 specification:
       *
       * "When aggregated into arrays within a shader, images can only be
       *  indexed with a constant integral expression."
       *
       * On the other hand the desktop GL specification extension allows
       * non-constant indexing of image arrays, but behavior is left undefined
       * in cases where the indexing expression is not dynamically uniform.
       */
      if (state->es_shader && array->type->without_array()->is_image()) {
         _mesa_glsl_error(&loc, state,
                          "image arrays indexed with non-constant "
                          "expressions are forbidden in GLSL ES.");
      }
   }

   /* After performing all of the error checking, generate the IR for the
    * expression.
    */
   if (array->type->is_array()
       || array->type->is_matrix()) {
      return new(mem_ctx) ir_dereference_array(array, idx);
   } else if (array->type->is_vector()) {
      return new(mem_ctx) ir_expression(ir_binop_vector_extract, array, idx);
   } else if (array->type->is_error()) {
      return array;
   } else {
      ir_rvalue *result = new(mem_ctx) ir_dereference_array(array, idx);
      result->type = glsl_type::error_type;

      return result;
   }
}
<|endoftext|>


<commit_before>/*
 * Copyright © 2010 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include "ast.h"
#include "glsl_types.h"
#include "ir.h"

void
ast_array_specifier::print(void) const
{
   if (this->is_unsized_array) {
      printf("[ ] ");
   }

   foreach_list_typed (ast_node, array_dimension, link, &this->array_dimensions) {
      printf("[ ");
      array_dimension->print();
      printf("] ");
   }
}

/**
 * If \c ir is a reference to an array for which we are tracking the max array
 * element accessed, track that the given element has been accessed.
 * Otherwise do nothing.
 *
 * This function also checks whether the array is a built-in array whose
 * maximum size is too small to accommodate the given index, and if so uses
 * loc and state to report the error.
 */
static void
update_max_array_access(ir_rvalue *ir, int idx, YYLTYPE *loc,
                        struct _mesa_glsl_parse_state *state)
{
   if (ir_dereference_variable *deref_var = ir->as_dereference_variable()) {
      ir_variable *var = deref_var->var;
      if (idx > (int)var->data.max_array_access) {
         var->data.max_array_access = idx;

         /* Check whether this access will, as a side effect, implicitly cause
          * the size of a built-in array to be too large.
          */
         check_builtin_array_max_size(var->name, idx+1, *loc, state);
      }
   } else if (ir_dereference_record *deref_record =
              ir->as_dereference_record()) {
      /* There are two possibilities we need to consider:
       *
       * - Accessing an element of an array that is a member of a named
       *   interface block (e.g. ifc.foo[i])
       *
       * - Accessing an element of an array that is a member of a named
       *   interface block array (e.g. ifc[j].foo[i]).
       */
      ir_dereference_variable *deref_var =
         deref_record->record->as_dereference_variable();
      if (deref_var == NULL) {
         if (ir_dereference_array *deref_array =
             deref_record->record->as_dereference_array()) {
            deref_var = deref_array->array->as_dereference_variable();
         }
      }

      if (deref_var != NULL) {
         if (deref_var->var->is_interface_instance()) {
            unsigned field_index =
               deref_record->record->type->field_index(deref_record->field);
            assert(field_index < deref_var->var->get_interface_type()->length);

            unsigned *const max_ifc_array_access =
               deref_var->var->get_max_ifc_array_access();

            assert(max_ifc_array_access != NULL);

            if (idx > (int)max_ifc_array_access[field_index]) {
               max_ifc_array_access[field_index] = idx;

               /* Check whether this access will, as a side effect, implicitly
                * cause the size of a built-in array to be too large.
                */
               check_builtin_array_max_size(deref_record->field, idx+1, *loc,
                                            state);
            }
         }
      }
   }
}

static int
get_implicit_array_size(struct _mesa_glsl_parse_state *state,
                        ir_rvalue *array)
{
   ir_variable *var = array->variable_referenced();

   /* Inputs in control shader are implicitly sized
    * to the maximum patch size.
    */
   if (state->stage == MESA_SHADER_TESS_CTRL &&
       var->data.mode == ir_var_shader_in) {
      return state->Const.MaxPatchVertices;
   }

   /* Non-patch inputs in evaluation shader are implicitly sized
    * to the maximum patch size.
    */
   if (state->stage == MESA_SHADER_TESS_EVAL &&
       var->data.mode == ir_var_shader_in &&
       !var->data.patch) {
      return state->Const.MaxPatchVertices;
   }

   return 0;
}

ir_rvalue *
_mesa_ast_array_index_to_hir(void *mem_ctx,
			     struct _mesa_glsl_parse_state *state,
			     ir_rvalue *array, ir_rvalue *idx,
			     YYLTYPE &loc, YYLTYPE &idx_loc)
{
   if (!array->type->is_error()
       && !array->type->is_array()
       && !array->type->is_matrix()
       && !array->type->is_vector()) {
      _mesa_glsl_error(& idx_loc, state,
		       "cannot dereference non-array / non-matrix / "
		       "non-vector");
   }

   if (!idx->type->is_error()) {
      if (!idx->type->is_integer()) {
	 _mesa_glsl_error(& idx_loc, state, "array index must be integer type");
      } else if (!idx->type->is_scalar()) {
	 _mesa_glsl_error(& idx_loc, state, "array index must be scalar");
      }
   }

   /* If the array index is a constant expression and the array has a
    * declared size, ensure that the access is in-bounds.  If the array
    * index is not a constant expression, ensure that the array has a
    * declared size.
    */
   ir_constant *const const_index = idx->constant_expression_value();
   if (const_index != NULL && idx->type->is_integer()) {
      const int idx = const_index->value.i[0];
      const char *type_name = "error";
      unsigned bound = 0;

      /* From page 24 (page 30 of the PDF) of the GLSL 1.50 spec:
       *
       *    "It is illegal to declare an array with a size, and then
       *    later (in the same shader) index the same array with an
       *    integral constant expression greater than or equal to the
       *    declared size. It is also illegal to index an array with a
       *    negative constant expression."
       */
      if (array->type->is_matrix()) {
	 if (array->type->row_type()->vector_elements <= idx) {
	    type_name = "matrix";
	    bound = array->type->row_type()->vector_elements;
	 }
      } else if (array->type->is_vector()) {
	 if (array->type->vector_elements <= idx) {
	    type_name = "vector";
	    bound = array->type->vector_elements;
	 }
      } else {
	 /* glsl_type::array_size() returns -1 for non-array types.  This means
	  * that we don't need to verify that the type is an array before
	  * doing the bounds checking.
	  */
	 if ((array->type->array_size() > 0)
	     && (array->type->array_size() <= idx)) {
	    type_name = "array";
	    bound = array->type->array_size();
	 }
      }

      if (bound > 0) {
	 _mesa_glsl_error(& loc, state, "%s index must be < %u",
			  type_name, bound);
      } else if (idx < 0) {
	 _mesa_glsl_error(& loc, state, "%s index must be >= 0",
			  type_name);
      }

      if (array->type->is_array())
         update_max_array_access(array, idx, &loc, state);
   } else if (const_index == NULL && array->type->is_array()) {
      if (array->type->is_unsized_array()) {
         int implicit_size = get_implicit_array_size(state, array);
         if (implicit_size) {
            ir_variable *v = array->whole_variable_referenced();
            if (v != NULL)
               v->data.max_array_access = implicit_size - 1;
         }
         else {
            _mesa_glsl_error(&loc, state, "unsized array index must be constant");
         }
      } else if (array->type->fields.array->is_interface()
                 && array->variable_referenced()->data.mode == ir_var_uniform
                 && !state->is_version(400, 0) && !state->ARB_gpu_shader5_enable) {
	 /* Page 46 in section 4.3.7 of the OpenGL ES 3.00 spec says:
	  *
	  *     "All indexes used to index a uniform block array must be
	  *     constant integral expressions."
	  */
	 _mesa_glsl_error(&loc, state,
			  "uniform block array index must be constant");
      } else {
	 /* whole_variable_referenced can return NULL if the array is a
	  * member of a structure.  In this case it is safe to not update
	  * the max_array_access field because it is never used for fields
	  * of structures.
	  */
	 ir_variable *v = array->whole_variable_referenced();
	 if (v != NULL)
	    v->data.max_array_access = array->type->array_size() - 1;
      }

      /* From page 23 (29 of the PDF) of the GLSL 1.30 spec:
       *
       *    "Samplers aggregated into arrays within a shader (using square
       *    brackets [ ]) can only be indexed with integral constant
       *    expressions [...]."
       *
       * This restriction was added in GLSL 1.30.  Shaders using earlier
       * version of the language should not be rejected by the compiler
       * front-end for using this construct.  This allows useful things such
       * as using a loop counter as the index to an array of samplers.  If the
       * loop in unrolled, the code should compile correctly.  Instead, emit a
       * warning.
       *
       * In GLSL 4.00 / ARB_gpu_shader5, this requirement is relaxed again to allow
       * indexing with dynamically uniform expressions. Note that these are not
       * required to be uniforms or expressions based on them, but merely that the
       * values must not diverge between shader invocations run together. If the
       * values *do* diverge, then the behavior of the operation requiring a
       * dynamically uniform expression is undefined.
       */
      if (array->type->without_array()->is_sampler()) {
         if (!state->is_version(400, 0) && !state->ARB_gpu_shader5_enable) {
            if (state->is_version(130, 300))
               _mesa_glsl_error(&loc, state,
                                "sampler arrays indexed with non-constant "
                                "expressions are forbidden in GLSL %s "
                                "and later",
                                state->es_shader ? "ES 3.00" : "1.30");
            else if (state->es_shader)
               _mesa_glsl_warning(&loc, state,
                                  "sampler arrays indexed with non-constant "
                                  "expressions will be forbidden in GLSL "
                                  "3.00 and later");
            else
               _mesa_glsl_warning(&loc, state,
                                  "sampler arrays indexed with non-constant "
                                  "expressions will be forbidden in GLSL "
                                  "1.30 and later");
         }
      }
   }

   /* After performing all of the error checking, generate the IR for the
    * expression.
    */
   if (array->type->is_array()
       || array->type->is_matrix()) {
      return new(mem_ctx) ir_dereference_array(array, idx);
   } else if (array->type->is_vector()) {
      return new(mem_ctx) ir_expression(ir_binop_vector_extract, array, idx);
   } else if (array->type->is_error()) {
      return array;
   } else {
      ir_rvalue *result = new(mem_ctx) ir_dereference_array(array, idx);
      result->type = glsl_type::error_type;

      return result;
   }
}
<commit_msg>glsl: allow indexing of gl_out with a non-const if length isn't known<commit_after>/*
 * Copyright © 2010 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include "ast.h"
#include "glsl_types.h"
#include "ir.h"

void
ast_array_specifier::print(void) const
{
   if (this->is_unsized_array) {
      printf("[ ] ");
   }

   foreach_list_typed (ast_node, array_dimension, link, &this->array_dimensions) {
      printf("[ ");
      array_dimension->print();
      printf("] ");
   }
}

/**
 * If \c ir is a reference to an array for which we are tracking the max array
 * element accessed, track that the given element has been accessed.
 * Otherwise do nothing.
 *
 * This function also checks whether the array is a built-in array whose
 * maximum size is too small to accommodate the given index, and if so uses
 * loc and state to report the error.
 */
static void
update_max_array_access(ir_rvalue *ir, int idx, YYLTYPE *loc,
                        struct _mesa_glsl_parse_state *state)
{
   if (ir_dereference_variable *deref_var = ir->as_dereference_variable()) {
      ir_variable *var = deref_var->var;
      if (idx > (int)var->data.max_array_access) {
         var->data.max_array_access = idx;

         /* Check whether this access will, as a side effect, implicitly cause
          * the size of a built-in array to be too large.
          */
         check_builtin_array_max_size(var->name, idx+1, *loc, state);
      }
   } else if (ir_dereference_record *deref_record =
              ir->as_dereference_record()) {
      /* There are two possibilities we need to consider:
       *
       * - Accessing an element of an array that is a member of a named
       *   interface block (e.g. ifc.foo[i])
       *
       * - Accessing an element of an array that is a member of a named
       *   interface block array (e.g. ifc[j].foo[i]).
       */
      ir_dereference_variable *deref_var =
         deref_record->record->as_dereference_variable();
      if (deref_var == NULL) {
         if (ir_dereference_array *deref_array =
             deref_record->record->as_dereference_array()) {
            deref_var = deref_array->array->as_dereference_variable();
         }
      }

      if (deref_var != NULL) {
         if (deref_var->var->is_interface_instance()) {
            unsigned field_index =
               deref_record->record->type->field_index(deref_record->field);
            assert(field_index < deref_var->var->get_interface_type()->length);

            unsigned *const max_ifc_array_access =
               deref_var->var->get_max_ifc_array_access();

            assert(max_ifc_array_access != NULL);

            if (idx > (int)max_ifc_array_access[field_index]) {
               max_ifc_array_access[field_index] = idx;

               /* Check whether this access will, as a side effect, implicitly
                * cause the size of a built-in array to be too large.
                */
               check_builtin_array_max_size(deref_record->field, idx+1, *loc,
                                            state);
            }
         }
      }
   }
}

static int
get_implicit_array_size(struct _mesa_glsl_parse_state *state,
                        ir_rvalue *array)
{
   ir_variable *var = array->variable_referenced();

   /* Inputs in control shader are implicitly sized
    * to the maximum patch size.
    */
   if (state->stage == MESA_SHADER_TESS_CTRL &&
       var->data.mode == ir_var_shader_in) {
      return state->Const.MaxPatchVertices;
   }

   /* Non-patch inputs in evaluation shader are implicitly sized
    * to the maximum patch size.
    */
   if (state->stage == MESA_SHADER_TESS_EVAL &&
       var->data.mode == ir_var_shader_in &&
       !var->data.patch) {
      return state->Const.MaxPatchVertices;
   }

   return 0;
}

ir_rvalue *
_mesa_ast_array_index_to_hir(void *mem_ctx,
			     struct _mesa_glsl_parse_state *state,
			     ir_rvalue *array, ir_rvalue *idx,
			     YYLTYPE &loc, YYLTYPE &idx_loc)
{
   if (!array->type->is_error()
       && !array->type->is_array()
       && !array->type->is_matrix()
       && !array->type->is_vector()) {
      _mesa_glsl_error(& idx_loc, state,
		       "cannot dereference non-array / non-matrix / "
		       "non-vector");
   }

   if (!idx->type->is_error()) {
      if (!idx->type->is_integer()) {
	 _mesa_glsl_error(& idx_loc, state, "array index must be integer type");
      } else if (!idx->type->is_scalar()) {
	 _mesa_glsl_error(& idx_loc, state, "array index must be scalar");
      }
   }

   /* If the array index is a constant expression and the array has a
    * declared size, ensure that the access is in-bounds.  If the array
    * index is not a constant expression, ensure that the array has a
    * declared size.
    */
   ir_constant *const const_index = idx->constant_expression_value();
   if (const_index != NULL && idx->type->is_integer()) {
      const int idx = const_index->value.i[0];
      const char *type_name = "error";
      unsigned bound = 0;

      /* From page 24 (page 30 of the PDF) of the GLSL 1.50 spec:
       *
       *    "It is illegal to declare an array with a size, and then
       *    later (in the same shader) index the same array with an
       *    integral constant expression greater than or equal to the
       *    declared size. It is also illegal to index an array with a
       *    negative constant expression."
       */
      if (array->type->is_matrix()) {
	 if (array->type->row_type()->vector_elements <= idx) {
	    type_name = "matrix";
	    bound = array->type->row_type()->vector_elements;
	 }
      } else if (array->type->is_vector()) {
	 if (array->type->vector_elements <= idx) {
	    type_name = "vector";
	    bound = array->type->vector_elements;
	 }
      } else {
	 /* glsl_type::array_size() returns -1 for non-array types.  This means
	  * that we don't need to verify that the type is an array before
	  * doing the bounds checking.
	  */
	 if ((array->type->array_size() > 0)
	     && (array->type->array_size() <= idx)) {
	    type_name = "array";
	    bound = array->type->array_size();
	 }
      }

      if (bound > 0) {
	 _mesa_glsl_error(& loc, state, "%s index must be < %u",
			  type_name, bound);
      } else if (idx < 0) {
	 _mesa_glsl_error(& loc, state, "%s index must be >= 0",
			  type_name);
      }

      if (array->type->is_array())
         update_max_array_access(array, idx, &loc, state);
   } else if (const_index == NULL && array->type->is_array()) {
      if (array->type->is_unsized_array()) {
         int implicit_size = get_implicit_array_size(state, array);
         if (implicit_size) {
            ir_variable *v = array->whole_variable_referenced();
            if (v != NULL)
               v->data.max_array_access = implicit_size - 1;
         }
         else if (state->stage == MESA_SHADER_TESS_CTRL &&
                  array->variable_referenced()->data.mode == ir_var_shader_out &&
                  !array->variable_referenced()->data.patch) {
            /* Tessellation control shader output non-patch arrays are
             * initially unsized. Despite that, they are allowed to be
             * indexed with a non-constant expression (typically
             * "gl_InvocationID"). The array size will be determined
             * by the linker.
             */
         }
         else {
            _mesa_glsl_error(&loc, state, "unsized array index must be constant");
         }
      } else if (array->type->fields.array->is_interface()
                 && array->variable_referenced()->data.mode == ir_var_uniform
                 && !state->is_version(400, 0) && !state->ARB_gpu_shader5_enable) {
	 /* Page 46 in section 4.3.7 of the OpenGL ES 3.00 spec says:
	  *
	  *     "All indexes used to index a uniform block array must be
	  *     constant integral expressions."
	  */
	 _mesa_glsl_error(&loc, state,
			  "uniform block array index must be constant");
      } else {
	 /* whole_variable_referenced can return NULL if the array is a
	  * member of a structure.  In this case it is safe to not update
	  * the max_array_access field because it is never used for fields
	  * of structures.
	  */
	 ir_variable *v = array->whole_variable_referenced();
	 if (v != NULL)
	    v->data.max_array_access = array->type->array_size() - 1;
      }

      /* From page 23 (29 of the PDF) of the GLSL 1.30 spec:
       *
       *    "Samplers aggregated into arrays within a shader (using square
       *    brackets [ ]) can only be indexed with integral constant
       *    expressions [...]."
       *
       * This restriction was added in GLSL 1.30.  Shaders using earlier
       * version of the language should not be rejected by the compiler
       * front-end for using this construct.  This allows useful things such
       * as using a loop counter as the index to an array of samplers.  If the
       * loop in unrolled, the code should compile correctly.  Instead, emit a
       * warning.
       *
       * In GLSL 4.00 / ARB_gpu_shader5, this requirement is relaxed again to allow
       * indexing with dynamically uniform expressions. Note that these are not
       * required to be uniforms or expressions based on them, but merely that the
       * values must not diverge between shader invocations run together. If the
       * values *do* diverge, then the behavior of the operation requiring a
       * dynamically uniform expression is undefined.
       */
      if (array->type->without_array()->is_sampler()) {
         if (!state->is_version(400, 0) && !state->ARB_gpu_shader5_enable) {
            if (state->is_version(130, 300))
               _mesa_glsl_error(&loc, state,
                                "sampler arrays indexed with non-constant "
                                "expressions are forbidden in GLSL %s "
                                "and later",
                                state->es_shader ? "ES 3.00" : "1.30");
            else if (state->es_shader)
               _mesa_glsl_warning(&loc, state,
                                  "sampler arrays indexed with non-constant "
                                  "expressions will be forbidden in GLSL "
                                  "3.00 and later");
            else
               _mesa_glsl_warning(&loc, state,
                                  "sampler arrays indexed with non-constant "
                                  "expressions will be forbidden in GLSL "
                                  "1.30 and later");
         }
      }
   }

   /* After performing all of the error checking, generate the IR for the
    * expression.
    */
   if (array->type->is_array()
       || array->type->is_matrix()) {
      return new(mem_ctx) ir_dereference_array(array, idx);
   } else if (array->type->is_vector()) {
      return new(mem_ctx) ir_expression(ir_binop_vector_extract, array, idx);
   } else if (array->type->is_error()) {
      return array;
   } else {
      ir_rvalue *result = new(mem_ctx) ir_dereference_array(array, idx);
      result->type = glsl_type::error_type;

      return result;
   }
}
<|endoftext|>


<commit_before><commit_msg>grt: skip guides on nets grt ignores, with warning<commit_after><|endoftext|>