<commit_before>/*
 * Copyright 2009-2018 Jose Fonseca
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

/*
 * Simple catchsegv like utility.
 */

#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>

#include <windows.h>
#include <dbghelp.h>

#include <getopt.h>

#include <string>

#include "log.h"
#include "debugger.h"
#include "symbols.h"

static void
outputCallback(const char *s)
{
    fputs(s, stderr);
    fflush(stderr);
}

static ULONG g_TimeOut = 0;
static HANDLE g_hTimer = NULL;
static HANDLE g_hTimerQueue = NULL;
static DWORD g_ElapsedTime = 0;
static BOOL g_TimerIgnore = FALSE;
static DWORD g_Period = 1000;

static void
TerminateProcessById(DWORD dwProcessId)
{
    BOOL bTerminated = FALSE;
    HANDLE hProcess = OpenProcess(PROCESS_TERMINATE, FALSE, dwProcessId);
    if (hProcess) {
        bTerminated = TerminateProcess(hProcess, 3);
        CloseHandle(hProcess);
    }
    if (!bTerminated) {
       fprintf(stderr, "catchsegv: error: failed to interrupt target (0x%08lx)\n", GetLastError());
       exit(EXIT_FAILURE);
    }
}

/*
 * Periodically scans the desktop for modal dialog windows.
 *
 * See also http://msdn.microsoft.com/en-us/library/ms940840.aspx
 */
static BOOL CALLBACK
EnumWindowCallback(HWND hWnd, LPARAM lParam)
{
    DWORD dwProcessId = 0;
    DWORD dwThreadId;

    dwThreadId = GetWindowThreadProcessId(hWnd, &dwProcessId);
    if (dwProcessId == (DWORD)lParam) {
        if (GetWindowLong(hWnd, GWL_STYLE) & DS_MODALFRAME) {
            char szWindowText[256];
            if (GetWindowTextA(hWnd, szWindowText, _countof(szWindowText)) <= 0) {
                szWindowText[0] = 0;
            }

            fprintf(stderr, "catchsegv: error: message dialog detected (%s)\n", szWindowText);

            g_TimerIgnore = TRUE;

            assert(dwThreadId != 0);

            TrapThread(dwProcessId, dwThreadId);

            return FALSE;
        }
    }

    return TRUE;
}

static VOID CALLBACK
TimeOutCallback(PVOID lpParam, BOOLEAN TimerOrWaitFired)
{
    DWORD dwProcessId = (DWORD)(UINT_PTR)lpParam;

    if (g_TimerIgnore) {
        return;
    }

    EnumWindows(EnumWindowCallback, (LPARAM)dwProcessId);

    if (g_TimerIgnore) {
        return;
    }

    g_ElapsedTime += g_Period;

    if (!g_TimeOut || g_ElapsedTime < g_TimeOut*1000) {
        return;
    }

    fprintf(stderr, "catchsegv: time out (%lu sec) exceeded\n", g_TimeOut);

    g_TimerIgnore = TRUE;

    TerminateProcessById(dwProcessId);
}

static void
Usage(void)
{
    fputs("usage: catchsegv [options] <command-line>\n"
          "\n"
          "options:\n"
          "  -?         displays command line help text\n"
          "  -v         enables verbose output from the debugger\n"
          "  -t SECONDS specifies a timeout in seconds \n"
          "  -1         dump stack on first chance exceptions \n"
          "  -H         use debug heap\n" ,
          stderr);
}

/*
 * Ignore Ctrl-C / Ctrl-Break events, so this process stays alive long enough
 * to dump the stack backtraces of the debuggee.  But honour the second event.
 */
static BOOL WINAPI
consoleCtrlHandler(DWORD fdwCtrlType)
{
    static int cCtrlC = 0;
    static int cCtrlBreak = 0;

    switch (fdwCtrlType) {
    case CTRL_C_EVENT:
        fprintf(stderr, "catchsegv: warning: caught Ctrl-C event\n");
        return cCtrlC++ ? FALSE : TRUE;
    case CTRL_BREAK_EVENT:
        fprintf(stderr, "catchsegv: warning: caught Ctrl-Break event\n");
        return cCtrlBreak++ ? FALSE : TRUE;
    default:
        return FALSE;
    }
}

/**
 * Determine whether an argument should be quoted.
 */
static bool
needsQuote(const char *arg)
{
    char c;
    while (true) {
        c = *arg++;
        if (c == '\0') {
            break;
        }
        if (c == ' ' || c == '\t' || c == '\"') {
            return true;
        }
        if (c == '\\') {
            c = *arg++;
            if (c == '\0') {
                break;
            }
            if (c == '"') {
                return true;
            }
        }
    }
    return false;
}

static void
quoteArg(std::string &s, const char *arg)
{
    char c;
    unsigned backslashes = 0;

    s.push_back('"');
    while (true) {
        c = *arg++;
        if (c == '\0') {
            break;
        } else if (c == '"') {
            while (backslashes) {
                s.push_back('\\');
                --backslashes;
            }
            s.push_back('\\');
        } else {
            if (c == '\\') {
                ++backslashes;
            } else {
                backslashes = 0;
            }
        }
        s.push_back(c);
    }
    s.push_back('"');
}

int
main(int argc, char** argv)
{
    /*
     * Disable error message boxes.
     */

#ifdef NDEBUG
    SetErrorMode(SEM_FAILCRITICALERRORS |
                 SEM_NOGPFAULTERRORBOX |
                 SEM_NOOPENFILEERRORBOX);

    // Disable assertion failure message box
    // http://msdn.microsoft.com/en-us/library/sas1dkb2.aspx
    _set_error_mode(_OUT_TO_STDERR);
#ifdef _MSC_VER
    // Disable abort message box
    // http://msdn.microsoft.com/en-us/library/e631wekh.aspx
    _set_abort_behavior(0, _WRITE_ABORT_MSG);
#endif
#endif

    /*
     * Parse command line arguments
     */

    bool debugHeap = false;
    while (1) {
        int opt = getopt(argc, argv, "?1dhHt:zv");

        switch (opt) {
        case 'h':
            Usage();
            return 0;
        case 'v':
            debugOptions.verbose_flag = true;
            break;
        case 'd':
            debugOptions.debug_flag = true;
            break;
        case '1':
            debugOptions.first_chance = true;
            break;
        case 't':
            g_TimeOut = strtoul(optarg, NULL, 0);
            break;
        case 'H':
            debugHeap = true;
            break;
        case 'z':
            debugOptions.minidump = true;
            break;
        case '?':
            if (optopt == '?') {
                Usage();
                return 0;
            }
            /* fall-trhough */
        default:
            opt = -1;
            break;
        }

        if (opt == -1) {
            break;
        }
    }

    /*
     * Concatenate remaining arguments into a command line
     */

    std::string commandLine;

    char sep = 0;
    while (optind < argc) {
        const char *arg = argv[optind];

        if (sep) {
            commandLine.push_back(sep);
        }

        if (needsQuote(arg)) {
            quoteArg(commandLine, arg);
        } else {
            commandLine.append(arg);
        }

        sep = ' ';

        ++optind;
    }

    if (commandLine.empty()) {
        fprintf(stderr, "catchsegv: error: no command line given\n\n");
        Usage();
        return EXIT_FAILURE;
    }

    setDumpCallback(&outputCallback);

    SetConsoleCtrlHandler(&consoleCtrlHandler, TRUE);

    // Disable debug heap
    // https://msdn.microsoft.com/en-us/library/windows/desktop/aa366705.aspx
    if (!debugHeap) {
        SetEnvironmentVariableA("_NO_DEBUG_HEAP", "1");
    }

    STARTUPINFOA StartupInfo;
    ZeroMemory(&StartupInfo, sizeof StartupInfo);
    StartupInfo.cb = sizeof StartupInfo;
    StartupInfo.dwFlags = STARTF_USESHOWWINDOW;
    StartupInfo.wShowWindow = SW_SHOWNORMAL;

    PROCESS_INFORMATION ProcessInformation;
    ZeroMemory(&ProcessInformation, sizeof ProcessInformation);

    if (!CreateProcessA(NULL, // lpApplicationName
                        const_cast<char *>(commandLine.c_str()),
                        NULL, // lpProcessAttributes
                        NULL, // lpThreadAttributes
                        TRUE, // bInheritHandles
                        DEBUG_PROCESS,
                        NULL, // lpEnvironment
                        NULL, // lpCurrentDirectory
                        &StartupInfo,
                        &ProcessInformation)) {
         fprintf(stderr, "catchsegv: error: failed to create the process (0x%08lx)\n", GetLastError());
         exit(EXIT_FAILURE);
    }

    DWORD dwProcessId = GetProcessId(ProcessInformation.hProcess);

    g_hTimerQueue = CreateTimerQueue();
    if (g_hTimerQueue == NULL) {
        fprintf(stderr, "catchsegv: error: failed to create a timer queue (0x%08lx)\n", GetLastError());
        return EXIT_FAILURE;
    }

    TIMECAPS tc;
    MMRESULT mmRes = timeGetDevCaps(&tc, sizeof tc);
    if (mmRes == MMSYSERR_NOERROR &&
        tc.wPeriodMax < g_Period) {
        g_Period = tc.wPeriodMax;
    }

    if (!CreateTimerQueueTimer(&g_hTimer, g_hTimerQueue,
                               (WAITORTIMERCALLBACK)TimeOutCallback,
                               (PVOID)(UINT_PTR)dwProcessId, g_Period, g_Period, 0)) {
        fprintf(stderr, "catchsegv: error: failed to CreateTimerQueueTimer failed (0x%08lx)\n", GetLastError());
        return EXIT_FAILURE;
    }

    /*
     * Set DbgHelp options
     */

    SetSymOptions(debugOptions.debug_flag);

    /*
     * Main event loop.
     */

    DebugMainLoop();

    DWORD dwExitCode = STILL_ACTIVE;
    GetExitCodeProcess(ProcessInformation.hProcess, &dwExitCode);

    CloseHandle(ProcessInformation.hProcess);
    CloseHandle(ProcessInformation.hThread);

    return dwExitCode;
}
<commit_msg>catchsegv: Documment all existing options.<commit_after>/*
 * Copyright 2009-2018 Jose Fonseca
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

/*
 * Simple catchsegv like utility.
 */

#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>

#include <windows.h>
#include <dbghelp.h>

#include <getopt.h>

#include <string>

#include "log.h"
#include "debugger.h"
#include "symbols.h"

static void
outputCallback(const char *s)
{
    fputs(s, stderr);
    fflush(stderr);
}

static ULONG g_TimeOut = 0;
static HANDLE g_hTimer = NULL;
static HANDLE g_hTimerQueue = NULL;
static DWORD g_ElapsedTime = 0;
static BOOL g_TimerIgnore = FALSE;
static DWORD g_Period = 1000;

static void
TerminateProcessById(DWORD dwProcessId)
{
    BOOL bTerminated = FALSE;
    HANDLE hProcess = OpenProcess(PROCESS_TERMINATE, FALSE, dwProcessId);
    if (hProcess) {
        bTerminated = TerminateProcess(hProcess, 3);
        CloseHandle(hProcess);
    }
    if (!bTerminated) {
       fprintf(stderr, "catchsegv: error: failed to interrupt target (0x%08lx)\n", GetLastError());
       exit(EXIT_FAILURE);
    }
}

/*
 * Periodically scans the desktop for modal dialog windows.
 *
 * See also http://msdn.microsoft.com/en-us/library/ms940840.aspx
 */
static BOOL CALLBACK
EnumWindowCallback(HWND hWnd, LPARAM lParam)
{
    DWORD dwProcessId = 0;
    DWORD dwThreadId;

    dwThreadId = GetWindowThreadProcessId(hWnd, &dwProcessId);
    if (dwProcessId == (DWORD)lParam) {
        if (GetWindowLong(hWnd, GWL_STYLE) & DS_MODALFRAME) {
            char szWindowText[256];
            if (GetWindowTextA(hWnd, szWindowText, _countof(szWindowText)) <= 0) {
                szWindowText[0] = 0;
            }

            fprintf(stderr, "catchsegv: error: message dialog detected (%s)\n", szWindowText);

            g_TimerIgnore = TRUE;

            assert(dwThreadId != 0);

            TrapThread(dwProcessId, dwThreadId);

            return FALSE;
        }
    }

    return TRUE;
}

static VOID CALLBACK
TimeOutCallback(PVOID lpParam, BOOLEAN TimerOrWaitFired)
{
    DWORD dwProcessId = (DWORD)(UINT_PTR)lpParam;

    if (g_TimerIgnore) {
        return;
    }

    EnumWindows(EnumWindowCallback, (LPARAM)dwProcessId);

    if (g_TimerIgnore) {
        return;
    }

    g_ElapsedTime += g_Period;

    if (!g_TimeOut || g_ElapsedTime < g_TimeOut*1000) {
        return;
    }

    fprintf(stderr, "catchsegv: time out (%lu sec) exceeded\n", g_TimeOut);

    g_TimerIgnore = TRUE;

    TerminateProcessById(dwProcessId);
}

static void
Usage(void)
{
    fputs("usage: catchsegv [options] <command-line>\n"
          "\n"
          "options:\n"
          "  -?|-h        displays command line help text\n"
          "  -v           enables verbose output from the debugger\n"
          "  -d           enables debugging output (for debugging catchsegv itself)\n"
          "  -t SECONDS   specifies a timeout in seconds\n"
          "  -1           dump stack on first chance exceptions\n"
          "  -z           write minidumps\n"
          "  -H           use debug heap\n",
          stderr);
}

/*
 * Ignore Ctrl-C / Ctrl-Break events, so this process stays alive long enough
 * to dump the stack backtraces of the debuggee.  But honour the second event.
 */
static BOOL WINAPI
consoleCtrlHandler(DWORD fdwCtrlType)
{
    static int cCtrlC = 0;
    static int cCtrlBreak = 0;

    switch (fdwCtrlType) {
    case CTRL_C_EVENT:
        fprintf(stderr, "catchsegv: warning: caught Ctrl-C event\n");
        return cCtrlC++ ? FALSE : TRUE;
    case CTRL_BREAK_EVENT:
        fprintf(stderr, "catchsegv: warning: caught Ctrl-Break event\n");
        return cCtrlBreak++ ? FALSE : TRUE;
    default:
        return FALSE;
    }
}

/**
 * Determine whether an argument should be quoted.
 */
static bool
needsQuote(const char *arg)
{
    char c;
    while (true) {
        c = *arg++;
        if (c == '\0') {
            break;
        }
        if (c == ' ' || c == '\t' || c == '\"') {
            return true;
        }
        if (c == '\\') {
            c = *arg++;
            if (c == '\0') {
                break;
            }
            if (c == '"') {
                return true;
            }
        }
    }
    return false;
}

static void
quoteArg(std::string &s, const char *arg)
{
    char c;
    unsigned backslashes = 0;

    s.push_back('"');
    while (true) {
        c = *arg++;
        if (c == '\0') {
            break;
        } else if (c == '"') {
            while (backslashes) {
                s.push_back('\\');
                --backslashes;
            }
            s.push_back('\\');
        } else {
            if (c == '\\') {
                ++backslashes;
            } else {
                backslashes = 0;
            }
        }
        s.push_back(c);
    }
    s.push_back('"');
}

int
main(int argc, char** argv)
{
    /*
     * Disable error message boxes.
     */

#ifdef NDEBUG
    SetErrorMode(SEM_FAILCRITICALERRORS |
                 SEM_NOGPFAULTERRORBOX |
                 SEM_NOOPENFILEERRORBOX);

    // Disable assertion failure message box
    // http://msdn.microsoft.com/en-us/library/sas1dkb2.aspx
    _set_error_mode(_OUT_TO_STDERR);
#ifdef _MSC_VER
    // Disable abort message box
    // http://msdn.microsoft.com/en-us/library/e631wekh.aspx
    _set_abort_behavior(0, _WRITE_ABORT_MSG);
#endif
#endif

    /*
     * Parse command line arguments
     */

    bool debugHeap = false;
    while (1) {
        int opt = getopt(argc, argv, "?1dhHt:zv");

        switch (opt) {
        case 'h':
            Usage();
            return 0;
        case 'v':
            debugOptions.verbose_flag = true;
            break;
        case 'd':
            debugOptions.debug_flag = true;
            break;
        case '1':
            debugOptions.first_chance = true;
            break;
        case 't':
            g_TimeOut = strtoul(optarg, NULL, 0);
            break;
        case 'H':
            debugHeap = true;
            break;
        case 'z':
            debugOptions.minidump = true;
            break;
        case '?':
            if (optopt == '?') {
                Usage();
                return 0;
            }
            /* fall-trhough */
        default:
            opt = -1;
            break;
        }

        if (opt == -1) {
            break;
        }
    }

    /*
     * Concatenate remaining arguments into a command line
     */

    std::string commandLine;

    char sep = 0;
    while (optind < argc) {
        const char *arg = argv[optind];

        if (sep) {
            commandLine.push_back(sep);
        }

        if (needsQuote(arg)) {
            quoteArg(commandLine, arg);
        } else {
            commandLine.append(arg);
        }

        sep = ' ';

        ++optind;
    }

    if (commandLine.empty()) {
        fprintf(stderr, "catchsegv: error: no command line given\n\n");
        Usage();
        return EXIT_FAILURE;
    }

    setDumpCallback(&outputCallback);

    SetConsoleCtrlHandler(&consoleCtrlHandler, TRUE);

    // Disable debug heap
    // https://msdn.microsoft.com/en-us/library/windows/desktop/aa366705.aspx
    if (!debugHeap) {
        SetEnvironmentVariableA("_NO_DEBUG_HEAP", "1");
    }

    STARTUPINFOA StartupInfo;
    ZeroMemory(&StartupInfo, sizeof StartupInfo);
    StartupInfo.cb = sizeof StartupInfo;
    StartupInfo.dwFlags = STARTF_USESHOWWINDOW;
    StartupInfo.wShowWindow = SW_SHOWNORMAL;

    PROCESS_INFORMATION ProcessInformation;
    ZeroMemory(&ProcessInformation, sizeof ProcessInformation);

    if (!CreateProcessA(NULL, // lpApplicationName
                        const_cast<char *>(commandLine.c_str()),
                        NULL, // lpProcessAttributes
                        NULL, // lpThreadAttributes
                        TRUE, // bInheritHandles
                        DEBUG_PROCESS,
                        NULL, // lpEnvironment
                        NULL, // lpCurrentDirectory
                        &StartupInfo,
                        &ProcessInformation)) {
         fprintf(stderr, "catchsegv: error: failed to create the process (0x%08lx)\n", GetLastError());
         exit(EXIT_FAILURE);
    }

    DWORD dwProcessId = GetProcessId(ProcessInformation.hProcess);

    g_hTimerQueue = CreateTimerQueue();
    if (g_hTimerQueue == NULL) {
        fprintf(stderr, "catchsegv: error: failed to create a timer queue (0x%08lx)\n", GetLastError());
        return EXIT_FAILURE;
    }

    TIMECAPS tc;
    MMRESULT mmRes = timeGetDevCaps(&tc, sizeof tc);
    if (mmRes == MMSYSERR_NOERROR &&
        tc.wPeriodMax < g_Period) {
        g_Period = tc.wPeriodMax;
    }

    if (!CreateTimerQueueTimer(&g_hTimer, g_hTimerQueue,
                               (WAITORTIMERCALLBACK)TimeOutCallback,
                               (PVOID)(UINT_PTR)dwProcessId, g_Period, g_Period, 0)) {
        fprintf(stderr, "catchsegv: error: failed to CreateTimerQueueTimer failed (0x%08lx)\n", GetLastError());
        return EXIT_FAILURE;
    }

    /*
     * Set DbgHelp options
     */

    SetSymOptions(debugOptions.debug_flag);

    /*
     * Main event loop.
     */

    DebugMainLoop();

    DWORD dwExitCode = STILL_ACTIVE;
    GetExitCodeProcess(ProcessInformation.hProcess, &dwExitCode);

    CloseHandle(ProcessInformation.hProcess);
    CloseHandle(ProcessInformation.hThread);

    return dwExitCode;
}
<|endoftext|>


<commit_before><commit_msg>Pass platform in start event<commit_after><|endoftext|>


<commit_before><commit_msg>forgot square root in RMS force<commit_after><|endoftext|>


<commit_before><commit_msg>ImGui::Text exported to lua<commit_after><|endoftext|>


<commit_before>/**
 * This file is part of the "libfnord" project
 *   Copyright (c) 2015 Paul Asmuth
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <chartsql/CSTableScan.h>
#include <chartsql/qtree/ColumnReferenceNode.h>
#include <chartsql/runtime/defaultruntime.h>
#include <chartsql/runtime/compile.h>
#include <stx/ieee754.h>

using namespace stx;

namespace csql {

CSTableScan::CSTableScan(
    RefPtr<SequentialScanNode> stmt,
    const String& cstable_filename,
    QueryBuilder* runtime) :
    stmt_(stmt),
    cstable_filename_(cstable_filename),
    runtime_(runtime),
    colindex_(0),
    aggr_strategy_(stmt_->aggregationStrategy()) {}

void CSTableScan::execute(
    ExecutionContext* context,
    Function<bool (int argc, const SValue* argv)> fn) {
  context->incrNumSubtasksTotal(1);

  Set<String> column_names;
  for (const auto& slnode : stmt_->selectList()) {
    findColumns(slnode->expression(), &column_names);
    column_names_.emplace_back(slnode->columnName());
  }

  auto where_expr = stmt_->whereExpression();
  if (!where_expr.isEmpty()) {
    findColumns(where_expr.get(), &column_names);
  }

  cstable::CSTableReader cstable(cstable_filename_);

  for (const auto& col : column_names) {
    if (cstable.hasColumn(col)) {
      columns_.emplace(
          col,
          ColumnRef(cstable.getColumnReader(col), colindex_++));
    }
  }

  for (auto& slnode : stmt_->selectList()) {
    resolveColumns(slnode->expression());
  }

  for (const auto& slnode : stmt_->selectList()) {
    select_list_.emplace_back(
        findMaxRepetitionLevel(slnode->expression()),
        runtime_->buildValueExpression(slnode->expression()),
        &scratch_);
  }

  if (!where_expr.isEmpty()) {
    resolveColumns(where_expr.get());
    where_expr_ = runtime_->buildValueExpression(where_expr.get());
  }

  if (columns_.empty()) {
    scanWithoutColumns(&cstable, fn);
  } else {
    scan(&cstable, fn);
  }

  context->incrNumSubtasksCompleted(1);
}

void CSTableScan::scan(
    cstable::CSTableReader* cstable,
    Function<bool (int argc, const SValue* argv)> fn) {
  uint64_t select_level = 0;
  uint64_t fetch_level = 0;

  Vector<SValue> in_row(colindex_, SValue{});
  Vector<SValue> out_row(select_list_.size(), SValue{});

  size_t num_records = 0;
  size_t total_records = cstable->numRecords();
  while (num_records < total_records) {
    uint64_t next_level = 0;

    if (fetch_level == 0) {
      ++num_records;
    }

    for (auto& col : columns_) {
      auto nextr = col.second.reader->nextRepetitionLevel();

      if (nextr >= fetch_level) {
        auto& reader = col.second.reader;

        uint64_t r;
        uint64_t d;
        void* data;
        size_t size;
        reader->next(&r, &d, &data, &size);

        if (d < reader->maxDefinitionLevel()) {
          in_row[col.second.index] = SValue();
        } else {
          switch (col.second.reader->type()) {

            case msg::FieldType::STRING:
              in_row[col.second.index] =
                  SValue(SValue::StringType((char*) data, size));
              break;

            case msg::FieldType::UINT32:
            case msg::FieldType::UINT64:
              switch (size) {
                case sizeof(uint32_t):
                  in_row[col.second.index] =
                      SValue(SValue::IntegerType(*((uint32_t*) data)));
                  break;
                case sizeof(uint64_t):
                  in_row[col.second.index] =
                      SValue(SValue::IntegerType(*((uint64_t*) data)));
                  break;
                case 0:
                  in_row[col.second.index] = SValue();
                  break;
              }
              break;

            case msg::FieldType::BOOLEAN:
              switch (size) {
                case sizeof(uint8_t):
                  in_row[col.second.index] =
                      SValue(SValue::BoolType(*((uint8_t*) data) > 0));
                  break;
                case sizeof(uint32_t):
                  in_row[col.second.index] =
                      SValue(SValue::BoolType(*((uint32_t*) data) > 0));
                  break;
                case sizeof(uint64_t):
                  in_row[col.second.index] =
                      SValue(SValue::BoolType(*((uint64_t*) data) > 0));
                  break;
                case 0:
                  in_row[col.second.index] = SValue(SValue::BoolType(false));
                  break;
              }
              break;

            case msg::FieldType::DOUBLE:
              switch (size) {
                case sizeof(double):
                  in_row[col.second.index] =
                      SValue(SValue::FloatType(*((double*) data)));
                  break;
                case 0:
                  in_row[col.second.index] = SValue();
                  break;
              }
              break;

            default:
              RAISE(kIllegalStateError);

          }
        }
      }

      next_level = std::max(
          next_level,
          col.second.reader->nextRepetitionLevel());
    }

    fetch_level = next_level;

    bool where_pred = true;
    if (where_expr_.get() != nullptr) {
      SValue where_tmp;
      where_expr_->evaluate(in_row.size(), in_row.data(), &where_tmp);
      where_pred = where_tmp.getBoolWithConversion();
    }

    if (where_pred) {
      for (int i = 0; i < select_list_.size(); ++i) {
        if (select_list_[i].rep_level >= select_level) {
          select_list_[i].compiled->accumulate(
              &select_list_[i].instance,
              in_row.size(),
              in_row.data());
        }
      }

      switch (aggr_strategy_) {

        case AggregationStrategy::AGGREGATE_ALL:
          break;

        case AggregationStrategy::AGGREGATE_WITHIN_RECORD:
          if (next_level != 0) {
            break;
          }

          for (int i = 0; i < select_list_.size(); ++i) {
            select_list_[i].compiled->result(
                &select_list_[i].instance,
                &out_row[i]);

            select_list_[i].compiled->reset(&select_list_[i].instance);
          }

          if (!fn(out_row.size(), out_row.data())) {
            return;
          }

          break;

        case AggregationStrategy::NO_AGGREGATION:
          for (int i = 0; i < select_list_.size(); ++i) {
            select_list_[i].compiled->evaluate(
                in_row.size(),
                in_row.data(),
                &out_row[i]);
          }

          if (!fn(out_row.size(), out_row.data())) {
            return;
          }

          break;

      }

      select_level = fetch_level;
    } else {
      select_level = std::min(select_level, fetch_level);
    }

    for (const auto& col : columns_) {
      if (col.second.reader->maxRepetitionLevel() >= select_level) {
        in_row[col.second.index] = SValue();
      }
    }
  }

  switch (aggr_strategy_) {
    case AggregationStrategy::AGGREGATE_ALL:
      for (int i = 0; i < select_list_.size(); ++i) {
        select_list_[i].compiled->result(
            &select_list_[i].instance,
            &out_row[i]);
      }

      fn(out_row.size(), out_row.data());
      break;

    default:
      break;

  }
}

void CSTableScan::scanWithoutColumns(
    cstable::CSTableReader* cstable,
    Function<bool (int argc, const SValue* argv)> fn) {
  Vector<SValue> out_row(select_list_.size(), SValue{});

  size_t total_records = cstable->numRecords();
  for (size_t i = 0; i < total_records; ++i) {
    bool where_pred = true;
    if (where_expr_.get() != nullptr) {
      SValue where_tmp;
      where_expr_->evaluate(0, nullptr, &where_tmp);
      where_pred = where_tmp.getBoolWithConversion();
    }

    if (where_pred) {
      switch (aggr_strategy_) {

        case AggregationStrategy::AGGREGATE_ALL:
          for (int i = 0; i < select_list_.size(); ++i) {
            select_list_[i].compiled->accumulate(
                &select_list_[i].instance,
                0,
                nullptr);
          }
          break;

        case AggregationStrategy::AGGREGATE_WITHIN_RECORD:
        case AggregationStrategy::NO_AGGREGATION:
          for (int i = 0; i < select_list_.size(); ++i) {
            select_list_[i].compiled->evaluate(
                0,
                nullptr,
                &out_row[i]);
          }

          if (!fn(out_row.size(), out_row.data())) {
            return;
          }
          break;
      }
    }
  }

  switch (aggr_strategy_) {
    case AggregationStrategy::AGGREGATE_ALL:
      for (int i = 0; i < select_list_.size(); ++i) {
        select_list_[i].compiled->result(
            &select_list_[i].instance,
            &out_row[i]);
      }

      fn(out_row.size(), out_row.data());
      break;

    default:
      break;

  }
}

void CSTableScan::findColumns(
    RefPtr<ValueExpressionNode> expr,
    Set<String>* column_names) const {
  auto fieldref = dynamic_cast<ColumnReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    column_names->emplace(fieldref->fieldName());
  }

  for (const auto& e : expr->arguments()) {
    findColumns(e, column_names);
  }
}

void CSTableScan::resolveColumns(RefPtr<ValueExpressionNode> expr) const {
  auto fieldref = dynamic_cast<ColumnReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    auto col = columns_.find(fieldref->fieldName());
    if (col == columns_.end()) {
      fieldref->setColumnIndex(size_t(-1));
    } else {
      fieldref->setColumnIndex(col->second.index);
    }
  }

  for (const auto& e : expr->arguments()) {
    resolveColumns(e);
  }
}

uint64_t CSTableScan::findMaxRepetitionLevel(
    RefPtr<ValueExpressionNode> expr) const {
  uint64_t max_level = 0;

  auto fieldref = dynamic_cast<ColumnReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    auto col = columns_.find(fieldref->fieldName());
      if (col != columns_.end()) {
      auto col_level = col->second.reader->maxRepetitionLevel();
      if (col_level > max_level) {
        max_level = col_level;
      }
    }
  }

  for (const auto& e : expr->arguments()) {
    auto e_level = findMaxRepetitionLevel(e);
    if (e_level > max_level) {
      max_level = e_level;
    }
  }

  return max_level;
}

Vector<String> CSTableScan::columnNames() const {
  return column_names_;
}

size_t CSTableScan::numColumns() const {
  return column_names_.size();
}

CSTableScan::ColumnRef::ColumnRef(
    RefPtr<cstable::ColumnReader> r,
    size_t i) :
    reader(r),
    index(i) {}

CSTableScan::ExpressionRef::ExpressionRef(
    size_t _rep_level,
    ScopedPtr<ValueExpression> _compiled,
    ScratchMemory* smem) :
    rep_level(_rep_level),
    compiled(std::move(_compiled)),
    instance(compiled->allocInstance(smem)) {}

CSTableScan::ExpressionRef::ExpressionRef(
    ExpressionRef&& other) :
    rep_level(other.rep_level),
    compiled(std::move(other.compiled)),
    instance(other.instance) {
  other.instance.scratch = nullptr;
}

CSTableScan::ExpressionRef::~ExpressionRef() {
  if (instance.scratch) {
    compiled->freeInstance(&instance);
  }
}

} // namespace csql
<commit_msg>logging<commit_after>/**
 * This file is part of the "libfnord" project
 *   Copyright (c) 2015 Paul Asmuth
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <chartsql/CSTableScan.h>
#include <chartsql/qtree/ColumnReferenceNode.h>
#include <chartsql/runtime/defaultruntime.h>
#include <chartsql/runtime/compile.h>
#include <stx/ieee754.h>
#include <stx/logging.h>

using namespace stx;

namespace csql {

CSTableScan::CSTableScan(
    RefPtr<SequentialScanNode> stmt,
    const String& cstable_filename,
    QueryBuilder* runtime) :
    stmt_(stmt),
    cstable_filename_(cstable_filename),
    runtime_(runtime),
    colindex_(0),
    aggr_strategy_(stmt_->aggregationStrategy()) {}

void CSTableScan::execute(
    ExecutionContext* context,
    Function<bool (int argc, const SValue* argv)> fn) {
  context->incrNumSubtasksTotal(1);

  logDebug("sql", "Scanning cstable: $0", cstable_filename_);

  Set<String> column_names;
  for (const auto& slnode : stmt_->selectList()) {
    findColumns(slnode->expression(), &column_names);
    column_names_.emplace_back(slnode->columnName());
  }

  auto where_expr = stmt_->whereExpression();
  if (!where_expr.isEmpty()) {
    findColumns(where_expr.get(), &column_names);
  }

  cstable::CSTableReader cstable(cstable_filename_);

  for (const auto& col : column_names) {
    if (cstable.hasColumn(col)) {
      columns_.emplace(
          col,
          ColumnRef(cstable.getColumnReader(col), colindex_++));
    }
  }

  for (auto& slnode : stmt_->selectList()) {
    resolveColumns(slnode->expression());
  }

  for (const auto& slnode : stmt_->selectList()) {
    select_list_.emplace_back(
        findMaxRepetitionLevel(slnode->expression()),
        runtime_->buildValueExpression(slnode->expression()),
        &scratch_);
  }

  if (!where_expr.isEmpty()) {
    resolveColumns(where_expr.get());
    where_expr_ = runtime_->buildValueExpression(where_expr.get());
  }

  if (columns_.empty()) {
    scanWithoutColumns(&cstable, fn);
  } else {
    scan(&cstable, fn);
  }

  context->incrNumSubtasksCompleted(1);
}

void CSTableScan::scan(
    cstable::CSTableReader* cstable,
    Function<bool (int argc, const SValue* argv)> fn) {
  uint64_t select_level = 0;
  uint64_t fetch_level = 0;

  Vector<SValue> in_row(colindex_, SValue{});
  Vector<SValue> out_row(select_list_.size(), SValue{});

  size_t num_records = 0;
  size_t total_records = cstable->numRecords();
  while (num_records < total_records) {
    uint64_t next_level = 0;

    if (fetch_level == 0) {
      ++num_records;
    }

    for (auto& col : columns_) {
      auto nextr = col.second.reader->nextRepetitionLevel();

      if (nextr >= fetch_level) {
        auto& reader = col.second.reader;

        uint64_t r;
        uint64_t d;
        void* data;
        size_t size;
        reader->next(&r, &d, &data, &size);

        if (d < reader->maxDefinitionLevel()) {
          in_row[col.second.index] = SValue();
        } else {
          switch (col.second.reader->type()) {

            case msg::FieldType::STRING:
              in_row[col.second.index] =
                  SValue(SValue::StringType((char*) data, size));
              break;

            case msg::FieldType::UINT32:
            case msg::FieldType::UINT64:
              switch (size) {
                case sizeof(uint32_t):
                  in_row[col.second.index] =
                      SValue(SValue::IntegerType(*((uint32_t*) data)));
                  break;
                case sizeof(uint64_t):
                  in_row[col.second.index] =
                      SValue(SValue::IntegerType(*((uint64_t*) data)));
                  break;
                case 0:
                  in_row[col.second.index] = SValue();
                  break;
              }
              break;

            case msg::FieldType::BOOLEAN:
              switch (size) {
                case sizeof(uint8_t):
                  in_row[col.second.index] =
                      SValue(SValue::BoolType(*((uint8_t*) data) > 0));
                  break;
                case sizeof(uint32_t):
                  in_row[col.second.index] =
                      SValue(SValue::BoolType(*((uint32_t*) data) > 0));
                  break;
                case sizeof(uint64_t):
                  in_row[col.second.index] =
                      SValue(SValue::BoolType(*((uint64_t*) data) > 0));
                  break;
                case 0:
                  in_row[col.second.index] = SValue(SValue::BoolType(false));
                  break;
              }
              break;

            case msg::FieldType::DOUBLE:
              switch (size) {
                case sizeof(double):
                  in_row[col.second.index] =
                      SValue(SValue::FloatType(*((double*) data)));
                  break;
                case 0:
                  in_row[col.second.index] = SValue();
                  break;
              }
              break;

            default:
              RAISE(kIllegalStateError);

          }
        }
      }

      next_level = std::max(
          next_level,
          col.second.reader->nextRepetitionLevel());
    }

    fetch_level = next_level;

    bool where_pred = true;
    if (where_expr_.get() != nullptr) {
      SValue where_tmp;
      where_expr_->evaluate(in_row.size(), in_row.data(), &where_tmp);
      where_pred = where_tmp.getBoolWithConversion();
    }

    if (where_pred) {
      for (int i = 0; i < select_list_.size(); ++i) {
        if (select_list_[i].rep_level >= select_level) {
          select_list_[i].compiled->accumulate(
              &select_list_[i].instance,
              in_row.size(),
              in_row.data());
        }
      }

      switch (aggr_strategy_) {

        case AggregationStrategy::AGGREGATE_ALL:
          break;

        case AggregationStrategy::AGGREGATE_WITHIN_RECORD:
          if (next_level != 0) {
            break;
          }

          for (int i = 0; i < select_list_.size(); ++i) {
            select_list_[i].compiled->result(
                &select_list_[i].instance,
                &out_row[i]);

            select_list_[i].compiled->reset(&select_list_[i].instance);
          }

          if (!fn(out_row.size(), out_row.data())) {
            return;
          }

          break;

        case AggregationStrategy::NO_AGGREGATION:
          for (int i = 0; i < select_list_.size(); ++i) {
            select_list_[i].compiled->evaluate(
                in_row.size(),
                in_row.data(),
                &out_row[i]);
          }

          if (!fn(out_row.size(), out_row.data())) {
            return;
          }

          break;

      }

      select_level = fetch_level;
    } else {
      select_level = std::min(select_level, fetch_level);
    }

    for (const auto& col : columns_) {
      if (col.second.reader->maxRepetitionLevel() >= select_level) {
        in_row[col.second.index] = SValue();
      }
    }
  }

  switch (aggr_strategy_) {
    case AggregationStrategy::AGGREGATE_ALL:
      for (int i = 0; i < select_list_.size(); ++i) {
        select_list_[i].compiled->result(
            &select_list_[i].instance,
            &out_row[i]);
      }

      fn(out_row.size(), out_row.data());
      break;

    default:
      break;

  }
}

void CSTableScan::scanWithoutColumns(
    cstable::CSTableReader* cstable,
    Function<bool (int argc, const SValue* argv)> fn) {
  Vector<SValue> out_row(select_list_.size(), SValue{});

  size_t total_records = cstable->numRecords();
  for (size_t i = 0; i < total_records; ++i) {
    bool where_pred = true;
    if (where_expr_.get() != nullptr) {
      SValue where_tmp;
      where_expr_->evaluate(0, nullptr, &where_tmp);
      where_pred = where_tmp.getBoolWithConversion();
    }

    if (where_pred) {
      switch (aggr_strategy_) {

        case AggregationStrategy::AGGREGATE_ALL:
          for (int i = 0; i < select_list_.size(); ++i) {
            select_list_[i].compiled->accumulate(
                &select_list_[i].instance,
                0,
                nullptr);
          }
          break;

        case AggregationStrategy::AGGREGATE_WITHIN_RECORD:
        case AggregationStrategy::NO_AGGREGATION:
          for (int i = 0; i < select_list_.size(); ++i) {
            select_list_[i].compiled->evaluate(
                0,
                nullptr,
                &out_row[i]);
          }

          if (!fn(out_row.size(), out_row.data())) {
            return;
          }
          break;
      }
    }
  }

  switch (aggr_strategy_) {
    case AggregationStrategy::AGGREGATE_ALL:
      for (int i = 0; i < select_list_.size(); ++i) {
        select_list_[i].compiled->result(
            &select_list_[i].instance,
            &out_row[i]);
      }

      fn(out_row.size(), out_row.data());
      break;

    default:
      break;

  }
}

void CSTableScan::findColumns(
    RefPtr<ValueExpressionNode> expr,
    Set<String>* column_names) const {
  auto fieldref = dynamic_cast<ColumnReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    column_names->emplace(fieldref->fieldName());
  }

  for (const auto& e : expr->arguments()) {
    findColumns(e, column_names);
  }
}

void CSTableScan::resolveColumns(RefPtr<ValueExpressionNode> expr) const {
  auto fieldref = dynamic_cast<ColumnReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    auto col = columns_.find(fieldref->fieldName());
    if (col == columns_.end()) {
      fieldref->setColumnIndex(size_t(-1));
    } else {
      fieldref->setColumnIndex(col->second.index);
    }
  }

  for (const auto& e : expr->arguments()) {
    resolveColumns(e);
  }
}

uint64_t CSTableScan::findMaxRepetitionLevel(
    RefPtr<ValueExpressionNode> expr) const {
  uint64_t max_level = 0;

  auto fieldref = dynamic_cast<ColumnReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    auto col = columns_.find(fieldref->fieldName());
      if (col != columns_.end()) {
      auto col_level = col->second.reader->maxRepetitionLevel();
      if (col_level > max_level) {
        max_level = col_level;
      }
    }
  }

  for (const auto& e : expr->arguments()) {
    auto e_level = findMaxRepetitionLevel(e);
    if (e_level > max_level) {
      max_level = e_level;
    }
  }

  return max_level;
}

Vector<String> CSTableScan::columnNames() const {
  return column_names_;
}

size_t CSTableScan::numColumns() const {
  return column_names_.size();
}

CSTableScan::ColumnRef::ColumnRef(
    RefPtr<cstable::ColumnReader> r,
    size_t i) :
    reader(r),
    index(i) {}

CSTableScan::ExpressionRef::ExpressionRef(
    size_t _rep_level,
    ScopedPtr<ValueExpression> _compiled,
    ScratchMemory* smem) :
    rep_level(_rep_level),
    compiled(std::move(_compiled)),
    instance(compiled->allocInstance(smem)) {}

CSTableScan::ExpressionRef::ExpressionRef(
    ExpressionRef&& other) :
    rep_level(other.rep_level),
    compiled(std::move(other.compiled)),
    instance(other.instance) {
  other.instance.scratch = nullptr;
}

CSTableScan::ExpressionRef::~ExpressionRef() {
  if (instance.scratch) {
    compiled->freeInstance(&instance);
  }
}

} // namespace csql
<|endoftext|>


<commit_before>/**
 * This file is part of the "libfnord" project
 *   Copyright (c) 2015 Paul Asmuth
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <chartsql/CSTableScan.h>
#include <chartsql/qtree/ColumnReferenceNode.h>
#include <chartsql/runtime/defaultruntime.h>
#include <chartsql/runtime/compile.h>
#include <fnord/inspect.h>

using namespace fnord;

namespace csql {

CSTableScan::CSTableScan(
    RefPtr<SequentialScanNode> stmt,
    cstable::CSTableReader&& cstable,
    DefaultRuntime* runtime) :
    cstable_(std::move(cstable)),
    colindex_(0),
    aggr_strategy_(stmt->aggregationStrategy()) {

  Set<String> column_names;
  for (const auto& slnode : stmt->selectList()) {
    findColumns(slnode->expression(), &column_names);
  }

  for (const auto& col : column_names) {
    columns_.emplace(col, ColumnRef(cstable_.getColumnReader(col), colindex_++));
  }

  for (auto& slnode : stmt->selectList()) {
    resolveColumns(slnode->expression());
  }

  for (const auto& slnode : stmt->selectList()) {
    select_list_.emplace_back(
        findMaxRepetitionLevel(slnode->expression()),
        runtime->buildScalarExpression(slnode->expression()),
        &scratch_);
  }
}

void CSTableScan::execute(
    ExecutionContext* context,
    Function<bool (int argc, const SValue* argv)> fn) {
  uint64_t select_level = 0;
  uint64_t fetch_level = 0;

  Vector<SValue> in_row(colindex_, SValue{});
  Vector<SValue> out_row(select_list_.size(), SValue{});

  size_t num_records = 0;
  size_t total_records = cstable_.numRecords();
  while (num_records < total_records) {
    uint64_t next_level = 0;

    if (fetch_level == 0) {
      ++num_records;
    }

    for (auto& col : columns_) {
      auto nextr = col.second.reader->nextRepetitionLevel();

      if (nextr >= fetch_level) {
        auto& reader = col.second.reader;

        uint64_t r;
        uint64_t d;
        void* data;
        size_t size;
        reader->next(&r, &d, &data, &size);

        switch (col.second.reader->type()) {

          case msg::FieldType::STRING:
            in_row[col.second.index] =
                SValue(SValue::StringType((char*) data, size));
            break;

          case msg::FieldType::UINT32:
          case msg::FieldType::UINT64:
            switch (size) {
              case sizeof(uint32_t):
                in_row[col.second.index] =
                    SValue(SValue::IntegerType(*((uint32_t*) data)));
                break;
              case sizeof(uint64_t):
                in_row[col.second.index] =
                    SValue(SValue::IntegerType(*((uint64_t*) data)));
                break;
              case 0:
                in_row[col.second.index] = SValue();
                break;
            }
            break;

          case msg::FieldType::BOOLEAN:
            switch (size) {
              case sizeof(uint32_t):
                in_row[col.second.index] =
                    SValue(SValue::BoolType(*((uint32_t*) data) > 0));
                break;
              case sizeof(uint64_t):
                in_row[col.second.index] =
                    SValue(SValue::BoolType(*((uint64_t*) data) > 0));
                break;
              case 0:
                in_row[col.second.index] = SValue(SValue::BoolType(false));
                break;
            }
            break;

          case msg::FieldType::OBJECT:
            RAISE(kIllegalStateError);

        }
      }

      next_level = std::max(next_level, col.second.reader->nextRepetitionLevel());
    }

    fetch_level = next_level;

    if (true) { // where clause
      for (int i = 0; i < select_list_.size(); ++i) {
        if (select_list_[i].rep_level >= select_level) {
          select_list_[i].compiled->accumulate(
              &select_list_[i].instance,
              in_row.size(),
              in_row.data());
        }
      }

      switch (aggr_strategy_) {

        case AggregationStrategy::AGGREGATE_ALL:
          break;

        case AggregationStrategy::AGGREGATE_WITHIN_RECORD:
          if (next_level != 0) {
            break;
          }
          /* fallthrough */

        case AggregationStrategy::NO_AGGREGATION:
          for (int i = 0; i < select_list_.size(); ++i) {
            select_list_[i].compiled->result(
                &select_list_[i].instance,
                &out_row[i]);

            select_list_[i].compiled->reset(&select_list_[i].instance);
          }

          if (!fn(out_row.size(), out_row.data())) {
            return;
          }

          break;

      }

      select_level = fetch_level;
    } else {
      select_level = std::min(select_level, fetch_level);
    }

    for (const auto& col : columns_) {
      if (col.second.reader->maxRepetitionLevel() >= select_level) {
        in_row[col.second.index] = SValue();
      }
    }
  }

  switch (aggr_strategy_) {
    case AggregationStrategy::AGGREGATE_ALL:
      for (int i = 0; i < select_list_.size(); ++i) {
        select_list_[i].compiled->result(
            &select_list_[i].instance,
            &out_row[i]);

        select_list_[i].compiled->reset(&select_list_[i].instance);
      }

      fn(out_row.size(), out_row.data());
      break;

    default:
      break;

  }
}

void CSTableScan::findColumns(
    RefPtr<ScalarExpressionNode> expr,
    Set<String>* column_names) const {
  auto fieldref = dynamic_cast<ColumnReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    column_names->emplace(fieldref->fieldName());
  }

  for (const auto& e : expr->arguments()) {
    findColumns(e, column_names);
  }
}

void CSTableScan::resolveColumns(RefPtr<ScalarExpressionNode> expr) const {
  auto fieldref = dynamic_cast<ColumnReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    auto col = columns_.find(fieldref->fieldName());
    if (col == columns_.end()) {
      RAISE(kIllegalStateError);
    }

    fieldref->setColumnIndex(col->second.index);
  }

  for (const auto& e : expr->arguments()) {
    resolveColumns(e);
  }
}

uint64_t CSTableScan::findMaxRepetitionLevel(
    RefPtr<ScalarExpressionNode> expr) const {
  uint64_t max_level = 0;

  auto fieldref = dynamic_cast<ColumnReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    auto col = columns_.find(fieldref->fieldName());
    if (col == columns_.end()) {
      RAISE(kIllegalStateError);
    }

    auto col_level = col->second.reader->maxRepetitionLevel();
    if (col_level > max_level) {
      max_level = col_level;
    }
  }

  for (const auto& e : expr->arguments()) {
    auto e_level = findMaxRepetitionLevel(e);
    if (e_level > max_level) {
      max_level = e_level;
    }
  }

  return max_level;
}

CSTableScan::ColumnRef::ColumnRef(
    RefPtr<cstable::ColumnReader> r,
    size_t i) :
    reader(r),
    index(i) {}

CSTableScan::ExpressionRef::ExpressionRef(
    size_t _rep_level,
    ScopedPtr<ScalarExpression> _compiled,
    ScratchMemory* smem) :
    rep_level(_rep_level),
    compiled(std::move(_compiled)),
    instance(compiled->allocInstance(smem)) {}

CSTableScan::ExpressionRef::ExpressionRef(
    ExpressionRef&& other) :
    rep_level(other.rep_level),
    compiled(std::move(other.compiled)),
    instance(other.instance) {
  other.instance.scratch = nullptr;
}

CSTableScan::ExpressionRef::~ExpressionRef() {
  if (instance.scratch) {
    compiled->freeInstance(&instance);
  }
}

} // namespace csql
<commit_msg>CSTableScan...<commit_after>/**
 * This file is part of the "libfnord" project
 *   Copyright (c) 2015 Paul Asmuth
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <chartsql/CSTableScan.h>
#include <chartsql/qtree/ColumnReferenceNode.h>
#include <chartsql/runtime/defaultruntime.h>
#include <chartsql/runtime/compile.h>
#include <fnord/inspect.h>

using namespace fnord;

namespace csql {

CSTableScan::CSTableScan(
    RefPtr<SequentialScanNode> stmt,
    cstable::CSTableReader&& cstable,
    DefaultRuntime* runtime) :
    cstable_(std::move(cstable)),
    colindex_(0),
    aggr_strategy_(stmt->aggregationStrategy()) {

  Set<String> column_names;
  for (const auto& slnode : stmt->selectList()) {
    findColumns(slnode->expression(), &column_names);
  }

  for (const auto& col : column_names) {
    columns_.emplace(col, ColumnRef(cstable_.getColumnReader(col), colindex_++));
  }

  for (auto& slnode : stmt->selectList()) {
    resolveColumns(slnode->expression());
  }

  for (const auto& slnode : stmt->selectList()) {
    select_list_.emplace_back(
        findMaxRepetitionLevel(slnode->expression()),
        runtime->buildScalarExpression(slnode->expression()),
        &scratch_);
  }
}

void CSTableScan::execute(
    ExecutionContext* context,
    Function<bool (int argc, const SValue* argv)> fn) {
  uint64_t select_level = 0;
  uint64_t fetch_level = 0;

  Vector<SValue> in_row(colindex_, SValue{});
  Vector<SValue> out_row(select_list_.size(), SValue{});

  size_t num_records = 0;
  size_t total_records = cstable_.numRecords();
  while (num_records < total_records) {
    uint64_t next_level = 0;

    if (fetch_level == 0) {
      ++num_records;
    }

    for (auto& col : columns_) {
      auto nextr = col.second.reader->nextRepetitionLevel();

      if (nextr >= fetch_level) {
        auto& reader = col.second.reader;

        uint64_t r;
        uint64_t d;
        void* data;
        size_t size;
        reader->next(&r, &d, &data, &size);

        switch (col.second.reader->type()) {

          case msg::FieldType::STRING:
            in_row[col.second.index] =
                SValue(SValue::StringType((char*) data, size));
            break;

          case msg::FieldType::UINT32:
          case msg::FieldType::UINT64:
            switch (size) {
              case sizeof(uint32_t):
                in_row[col.second.index] =
                    SValue(SValue::IntegerType(*((uint32_t*) data)));
                break;
              case sizeof(uint64_t):
                in_row[col.second.index] =
                    SValue(SValue::IntegerType(*((uint64_t*) data)));
                break;
              case 0:
                in_row[col.second.index] = SValue();
                break;
            }
            break;

          case msg::FieldType::BOOLEAN:
            switch (size) {
              case sizeof(uint32_t):
                in_row[col.second.index] =
                    SValue(SValue::BoolType(*((uint32_t*) data) > 0));
                break;
              case sizeof(uint64_t):
                in_row[col.second.index] =
                    SValue(SValue::BoolType(*((uint64_t*) data) > 0));
                break;
              case 0:
                in_row[col.second.index] = SValue(SValue::BoolType(false));
                break;
            }
            break;

          case msg::FieldType::OBJECT:
            RAISE(kIllegalStateError);

        }
      }

      next_level = std::max(
          next_level,
          col.second.reader->nextRepetitionLevel());
    }

    fetch_level = next_level;

    if (true) { // where clause
      for (int i = 0; i < select_list_.size(); ++i) {
        if (select_list_[i].rep_level >= select_level) {
          select_list_[i].compiled->accumulate(
              &select_list_[i].instance,
              in_row.size(),
              in_row.data());
        }
      }

      switch (aggr_strategy_) {

        case AggregationStrategy::AGGREGATE_ALL:
          break;

        case AggregationStrategy::AGGREGATE_WITHIN_RECORD:
          if (next_level != 0) {
            break;
          }
          /* fallthrough */

        case AggregationStrategy::NO_AGGREGATION:
          for (int i = 0; i < select_list_.size(); ++i) {
            select_list_[i].compiled->evaluate(
                in_row.size(),
                in_row.data(),
                &out_row[i]);
          }

          if (!fn(out_row.size(), out_row.data())) {
            return;
          }

          break;

      }

      select_level = fetch_level;
    } else {
      select_level = std::min(select_level, fetch_level);
    }
  }

  switch (aggr_strategy_) {
    case AggregationStrategy::AGGREGATE_ALL:
      for (int i = 0; i < select_list_.size(); ++i) {
        select_list_[i].compiled->result(
            &select_list_[i].instance,
            &out_row[i]);

        select_list_[i].compiled->reset(&select_list_[i].instance);
      }

      fn(out_row.size(), out_row.data());
      break;

    default:
      break;

  }
}

void CSTableScan::findColumns(
    RefPtr<ScalarExpressionNode> expr,
    Set<String>* column_names) const {
  auto fieldref = dynamic_cast<ColumnReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    column_names->emplace(fieldref->fieldName());
  }

  for (const auto& e : expr->arguments()) {
    findColumns(e, column_names);
  }
}

void CSTableScan::resolveColumns(RefPtr<ScalarExpressionNode> expr) const {
  auto fieldref = dynamic_cast<ColumnReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    auto col = columns_.find(fieldref->fieldName());
    if (col == columns_.end()) {
      RAISE(kIllegalStateError);
    }

    fieldref->setColumnIndex(col->second.index);
  }

  for (const auto& e : expr->arguments()) {
    resolveColumns(e);
  }
}

uint64_t CSTableScan::findMaxRepetitionLevel(
    RefPtr<ScalarExpressionNode> expr) const {
  uint64_t max_level = 0;

  auto fieldref = dynamic_cast<ColumnReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    auto col = columns_.find(fieldref->fieldName());
    if (col == columns_.end()) {
      RAISE(kIllegalStateError);
    }

    auto col_level = col->second.reader->maxRepetitionLevel();
    if (col_level > max_level) {
      max_level = col_level;
    }
  }

  for (const auto& e : expr->arguments()) {
    auto e_level = findMaxRepetitionLevel(e);
    if (e_level > max_level) {
      max_level = e_level;
    }
  }

  return max_level;
}

CSTableScan::ColumnRef::ColumnRef(
    RefPtr<cstable::ColumnReader> r,
    size_t i) :
    reader(r),
    index(i) {}

CSTableScan::ExpressionRef::ExpressionRef(
    size_t _rep_level,
    ScopedPtr<ScalarExpression> _compiled,
    ScratchMemory* smem) :
    rep_level(_rep_level),
    compiled(std::move(_compiled)),
    instance(compiled->allocInstance(smem)) {}

CSTableScan::ExpressionRef::ExpressionRef(
    ExpressionRef&& other) :
    rep_level(other.rep_level),
    compiled(std::move(other.compiled)),
    instance(other.instance) {
  other.instance.scratch = nullptr;
}

CSTableScan::ExpressionRef::~ExpressionRef() {
  if (instance.scratch) {
    compiled->freeInstance(&instance);
  }
}

} // namespace csql
<|endoftext|>


<commit_before>/*-------------------------------------------------------------------------
 *
 * expr_transformer.cpp
 * file description
 *
 * Copyright(c) 2015, CMU
 *
 *-------------------------------------------------------------------------
 */

#include <unordered_map>

#include "nodes/pprint.h"
#include "utils/rel.h"
#include "utils/lsyscache.h"
#include "executor/executor.h"
#include "parser/parsetree.h"

#include "backend/bridge/dml/tuple/tuple_transformer.h"
#include "backend/bridge/dml/expr/pg_func_map.h"
#include "backend/bridge/dml/expr/expr_transformer.h"
#include "backend/common/logger.h"
#include "backend/common/value.h"
#include "backend/common/value_factory.h"
#include "backend/expression/expression_util.h"

namespace peloton {
namespace bridge {

expression::AbstractExpression*
ReMapPgFunc(Oid pg_func_id,
            expression::AbstractExpression* lc,
            expression::AbstractExpression* rc);

extern std::unordered_map<Oid, peloton::ExpressionType> pg_func_map;

void ExprTransformer::PrintPostgressExprTree(const ExprState* expr_state,
                                             std::string prefix) {
  auto tag = nodeTag(expr_state->expr);

  /* TODO Not complete.
   * Not all ExprState has a child / children list,
   * so it would take some multiplexing to print it recursively here.
   */
  LOG_INFO("%u ", tag);
}

/**
 * @brief Transform a ExprState tree (Postgres) to a AbstractExpression tree (Peloton) recursively.
 */
expression::AbstractExpression* ExprTransformer::TransformExpr(
    const ExprState* expr_state) {

  if(!expr_state)
    return nullptr;

  /* Special case:
   * Input is a list of expressions.
   * Transform it to a conjunction tree.
   */
  if(expr_state->type == T_List) {
    return TransformList(expr_state);
  }

  expression::AbstractExpression* peloton_expr = nullptr;

  switch (nodeTag(expr_state->expr)) {
    case T_Const:
      peloton_expr = TransformConst(expr_state);
      break;

    case T_OpExpr:
      peloton_expr = TransformOp(expr_state);
      break;

    case T_Var:
      peloton_expr = TransformVar(expr_state);
      break;

    case T_BoolExpr:
      peloton_expr = TransformBool(expr_state);
      break;

    case T_Param:
      peloton_expr = TransformParam(expr_state);
      break;

    case T_RelabelType:
      peloton_expr = TransformRelabelType(expr_state);
      break;

    case T_FuncExpr:
      peloton_expr = TransformFunc(expr_state);
      break;

    default:
      LOG_ERROR("Unsupported Postgres Expr type: %u (see 'nodes.h')\n",
                nodeTag(expr_state->expr));
  }

  return peloton_expr;
}

bool ExprTransformer::CleanExprTree(expression::AbstractExpression* root) {

  // AbstractExpression's destructor already handles deleting children
  delete root;

  return true;
}

expression::AbstractExpression* ExprTransformer::TransformConst(
    const ExprState* es) {

  auto const_expr = reinterpret_cast<const Const*>(es->expr);

  Value value;

  if (const_expr->constisnull) {  // Constant is null
    value = ValueFactory::GetNullValue();
  }
  else if(const_expr->constbyval){  // non null
    value = TupleTransformer::GetValue(const_expr->constvalue,
                                       const_expr->consttype);
  }
  else if(const_expr->constlen == -1) {
    LOG_INFO("Probably handing a string constant \n");
    value = TupleTransformer::GetValue(const_expr->constvalue,
                                       const_expr->consttype);
  }
  else {
    LOG_ERROR("Unknown Const profile: constlen = %d , constbyval = %d, constvalue = %u \n",
              const_expr->constlen, const_expr->constbyval, const_expr->constvalue);
  }

  LOG_INFO("Const : ");
  std::cout << value << std::endl;

  // A Const Expr has no children.
  return expression::ConstantValueFactory(value);
}

expression::AbstractExpression* ExprTransformer::TransformOp(
    const ExprState* es) {

  LOG_INFO("Transform Op \n");

  auto op_expr = reinterpret_cast<const OpExpr*>(es->expr);
  auto func_state = reinterpret_cast<const FuncExprState*>(es);

  assert(op_expr->opfuncid != 0); // Hopefully it has been filled in by PG planner
  assert(list_length(func_state->args) <= 2);   // Hopefully it has at most two parameters

  expression::AbstractExpression* lc = nullptr;
  expression::AbstractExpression* rc = nullptr;

  // Add function arguments as children
  int i = 0;
  ListCell   *arg;
  foreach(arg, func_state->args)
  {
    ExprState  *argstate = (ExprState *) lfirst(arg);

    if(i == 0)
      lc = TransformExpr(argstate);
    else if(i == 1)
      rc = TransformExpr(argstate);
    else
      break; // skip >2 arguments

    i++;
  }

  return ReMapPgFunc(op_expr->opfuncid, lc, rc);
}

expression::AbstractExpression* ExprTransformer::TransformVar(
    const ExprState* es) {

  // Var expr only needs default ES
  auto var_expr = reinterpret_cast<const Var*>(es->expr);

  assert(var_expr->varattno != InvalidAttrNumber);

  oid_t tuple_idx = (var_expr->varno == INNER_VAR ? 1 : 0);  // Seems reasonable, c.f. ExecEvalScalarVarFast()
  oid_t value_idx = static_cast<oid_t>(var_expr->varattno - 1); // Damnit attno is 1-index

  LOG_INFO("tuple_idx = %u , value_idx = %u \n", tuple_idx, value_idx);

  // TupleValue expr has no children.
  return expression::TupleValueFactory(tuple_idx, value_idx);

}

expression::AbstractExpression* ExprTransformer::TransformBool(
    const ExprState* es) {

  auto bool_expr = reinterpret_cast<const BoolExpr*>(es->expr);
  auto bool_state = reinterpret_cast<const BoolExprState*>(es);

  auto bool_op = bool_expr->boolop;

  /*
   * AND and OR can take >=2 arguments,
   * while NOT should take only one.
   */
  assert(bool_state->args);
  assert(bool_op != NOT_EXPR || list_length(bool_state->args) == 1);
  assert(bool_op == NOT_EXPR || list_length(bool_state->args) >= 2);

  auto args = bool_state->args;

  switch(bool_op){

    case AND_EXPR:
      LOG_INFO("Bool AND list \n");
      return TransformList(reinterpret_cast<const ExprState*>(args), EXPRESSION_TYPE_CONJUNCTION_AND);

    case OR_EXPR:
      LOG_INFO("Bool OR list \n");
      return TransformList(reinterpret_cast<const ExprState*>(args), EXPRESSION_TYPE_CONJUNCTION_OR);

    case NOT_EXPR:
    {
      LOG_INFO("Bool NOT \n");
      auto child_es = reinterpret_cast<const ExprState*>(lfirst(list_head(args)));
      auto child = TransformExpr(child_es);
      return expression::OperatorFactory(EXPRESSION_TYPE_OPERATOR_NOT, child, nullptr);
    }

    default:
      LOG_ERROR("Unrecognized BoolExpr : %u", bool_op);
  }

  return nullptr;
}

expression::AbstractExpression* ExprTransformer::TransformParam(const ExprState *es) {

  auto param_expr = reinterpret_cast<const Param*>(es->expr);

  switch (param_expr->paramkind) {
    case PARAM_EXTERN:
      LOG_INFO("Handle EXTREN PARAM");
      return expression::ParameterValueFactory(param_expr->paramid + 1); // 1 indexed
      break;
    default:
      LOG_ERROR("Unrecognized param kind %d", param_expr->paramkind);
      break;
  }

  return nullptr;
}

expression::AbstractExpression* ExprTransformer::TransformRelabelType(const ExprState *es) {
  auto state = reinterpret_cast<const GenericExprState*>(es);
  auto expr = reinterpret_cast<const RelabelType *>(es->expr);
  auto child_state = state->arg;
  auto child_expr = expr->arg;

  assert(expr->xpr.type == T_RelabelType);

  LOG_INFO("child is of type %d, %d", child_expr->type, child_state->type);
  LOG_INFO("%d, %d", expr->resulttype, expr->relabelformat);
  expression::AbstractExpression *child = ExprTransformer::TransformExpr(child_state);

  //TODO: not implemented yet

  return child;
}

expression::AbstractExpression* ExprTransformer::TransformFunc(const ExprState *es) {
  auto state = reinterpret_cast<const FuncExprState*>(es);
  auto expr = reinterpret_cast<const FuncExpr*>(es->expr);
  auto expr_args = reinterpret_cast<const ExprState*>(state->args);

  assert(expr->xpr.type == T_FuncExpr);

  LOG_INFO("Return type: %d, isReturn %d, Coercion: %d", expr->funcresulttype, expr->funcretset, expr->funcformat);
  expression::AbstractExpression *args = ExprTransformer::TransformExpr(expr_args);
  LOG_INFO("args : %s", args->DebugInfo(" ").c_str());

  //TODO: not implemented yet

  return nullptr;

}

expression::AbstractExpression*
ExprTransformer::TransformList(const ExprState* es, ExpressionType et) {

  assert(et == EXPRESSION_TYPE_CONJUNCTION_AND || et == EXPRESSION_TYPE_CONJUNCTION_OR);

  const List* list = reinterpret_cast<const List*>(es);
  ListCell *l;
  int length = list_length(list);
  assert(length > 0);
  LOG_INFO("Expression List of length %d", length);
  std::list<expression::AbstractExpression*> exprs; // a list of AND'ed expressions

  foreach(l, list)
  {
    const ExprState *expr_state = reinterpret_cast<const ExprState*>(lfirst(l));
    exprs.push_back(ExprTransformer::TransformExpr(expr_state));
  }

  return expression::ConjunctionFactory(et, exprs);

}

/**
 * @brief Helper function: re-map Postgres's builtin function Oid
 * to proper expression type in Peloton
 *
 * @param pg_func_id  PG Function Id used to lookup function in \b fmrg_builtin[]
 * (see Postgres source file 'fmgrtab.cpp')
 * @param lc          Left child.
 * @param rc          Right child.
 * @return            Corresponding expression tree in peloton.
 */
expression::AbstractExpression*
ReMapPgFunc(Oid func_id,
            expression::AbstractExpression* lc,
            expression::AbstractExpression* rc) {

  auto itr = pg_func_map.find(func_id);

  if(itr == pg_func_map.end()){
    LOG_ERROR("Unsupported PG Function ID : %u (check fmgrtab.cpp)\n", func_id);
    return nullptr;
  }

  auto pl_expr_type = itr->second;

  switch(pl_expr_type){

    case EXPRESSION_TYPE_COMPARE_EQ:
    case EXPRESSION_TYPE_COMPARE_NE:
    case EXPRESSION_TYPE_COMPARE_GT:
    case EXPRESSION_TYPE_COMPARE_LT:
    case EXPRESSION_TYPE_COMPARE_GTE:
    case EXPRESSION_TYPE_COMPARE_LTE:
      return expression::ComparisonFactory(pl_expr_type, lc, rc);

    case EXPRESSION_TYPE_OPERATOR_PLUS:
    case EXPRESSION_TYPE_OPERATOR_MINUS:
    case EXPRESSION_TYPE_OPERATOR_MULTIPLY:
    case EXPRESSION_TYPE_OPERATOR_DIVIDE:
      return expression::OperatorFactory(pl_expr_type, lc, rc);

    default:
      LOG_ERROR("This Peloton ExpressionType is in our map but not transformed here : %u", pl_expr_type);
  }

  return nullptr;
}

}  // namespace bridge
}  // namespace peloton
<commit_msg>fix compilation<commit_after>/*-------------------------------------------------------------------------
 *
 * expr_transformer.cpp
 * file description
 *
 * Copyright(c) 2015, CMU
 *
 *-------------------------------------------------------------------------
 */

#include <unordered_map>

#include "nodes/pprint.h"
#include "utils/rel.h"
#include "utils/lsyscache.h"
#include "executor/executor.h"
#include "parser/parsetree.h"

#include "backend/bridge/dml/tuple/tuple_transformer.h"
#include "backend/bridge/dml/expr/pg_func_map.h"
#include "backend/bridge/dml/expr/expr_transformer.h"
#include "backend/common/logger.h"
#include "backend/common/value.h"
#include "backend/common/value_factory.h"
#include "backend/expression/expression_util.h"

namespace peloton {
namespace bridge {

expression::AbstractExpression*
ReMapPgFunc(Oid pg_func_id,
            expression::AbstractExpression* lc,
            expression::AbstractExpression* rc);

extern std::unordered_map<Oid, peloton::ExpressionType> pg_func_map;

void ExprTransformer::PrintPostgressExprTree(const ExprState* expr_state,
                                             std::string prefix) {
  auto tag = nodeTag(expr_state->expr);

  /* TODO Not complete.
   * Not all ExprState has a child / children list,
   * so it would take some multiplexing to print it recursively here.
   */
  LOG_INFO("%u ", tag);
}

/**
 * @brief Transform a ExprState tree (Postgres) to a AbstractExpression tree (Peloton) recursively.
 */
expression::AbstractExpression* ExprTransformer::TransformExpr(
    const ExprState* expr_state) {

  if(!expr_state)
    return nullptr;

  /* Special case:
   * Input is a list of expressions.
   * Transform it to a conjunction tree.
   */
  if(expr_state->type == T_List) {
    return TransformList(expr_state);
  }

  expression::AbstractExpression* peloton_expr = nullptr;

  switch (nodeTag(expr_state->expr)) {
    case T_Const:
      peloton_expr = TransformConst(expr_state);
      break;

    case T_OpExpr:
      peloton_expr = TransformOp(expr_state);
      break;

    case T_Var:
      peloton_expr = TransformVar(expr_state);
      break;

    case T_BoolExpr:
      peloton_expr = TransformBool(expr_state);
      break;

    case T_Param:
      peloton_expr = TransformParam(expr_state);
      break;

    case T_RelabelType:
      peloton_expr = TransformRelabelType(expr_state);
      break;

    case T_FuncExpr:
      peloton_expr = TransformFunc(expr_state);
      break;

    default:
      LOG_ERROR("Unsupported Postgres Expr type: %u (see 'nodes.h')\n",
                nodeTag(expr_state->expr));
  }

  return peloton_expr;
}

bool ExprTransformer::CleanExprTree(expression::AbstractExpression* root) {

  // AbstractExpression's destructor already handles deleting children
  delete root;

  return true;
}

expression::AbstractExpression* ExprTransformer::TransformConst(
    const ExprState* es) {

  auto const_expr = reinterpret_cast<const Const*>(es->expr);

  Value value;

  if (const_expr->constisnull) {  // Constant is null
    value = ValueFactory::GetNullValue();
  }
  else if(const_expr->constbyval){  // non null
    value = TupleTransformer::GetValue(const_expr->constvalue,
                                       const_expr->consttype);
  }
  else if(const_expr->constlen == -1) {
    LOG_INFO("Probably handing a string constant \n");
    value = TupleTransformer::GetValue(const_expr->constvalue,
                                       const_expr->consttype);
  }
  else {
    LOG_ERROR("Unknown Const profile: constlen = %d , constbyval = %d, constvalue = %lu \n",
              const_expr->constlen, const_expr->constbyval, (long unsigned)const_expr->constvalue);
  }

  LOG_INFO("Const : ");
  std::cout << value << std::endl;

  // A Const Expr has no children.
  return expression::ConstantValueFactory(value);
}

expression::AbstractExpression* ExprTransformer::TransformOp(
    const ExprState* es) {

  LOG_INFO("Transform Op \n");

  auto op_expr = reinterpret_cast<const OpExpr*>(es->expr);
  auto func_state = reinterpret_cast<const FuncExprState*>(es);

  assert(op_expr->opfuncid != 0); // Hopefully it has been filled in by PG planner
  assert(list_length(func_state->args) <= 2);   // Hopefully it has at most two parameters

  expression::AbstractExpression* lc = nullptr;
  expression::AbstractExpression* rc = nullptr;

  // Add function arguments as children
  int i = 0;
  ListCell   *arg;
  foreach(arg, func_state->args)
  {
    ExprState  *argstate = (ExprState *) lfirst(arg);

    if(i == 0)
      lc = TransformExpr(argstate);
    else if(i == 1)
      rc = TransformExpr(argstate);
    else
      break; // skip >2 arguments

    i++;
  }

  return ReMapPgFunc(op_expr->opfuncid, lc, rc);
}

expression::AbstractExpression* ExprTransformer::TransformVar(
    const ExprState* es) {

  // Var expr only needs default ES
  auto var_expr = reinterpret_cast<const Var*>(es->expr);

  assert(var_expr->varattno != InvalidAttrNumber);

  oid_t tuple_idx = (var_expr->varno == INNER_VAR ? 1 : 0);  // Seems reasonable, c.f. ExecEvalScalarVarFast()
  oid_t value_idx = static_cast<oid_t>(var_expr->varattno - 1); // Damnit attno is 1-index

  LOG_INFO("tuple_idx = %u , value_idx = %u \n", tuple_idx, value_idx);

  // TupleValue expr has no children.
  return expression::TupleValueFactory(tuple_idx, value_idx);

}

expression::AbstractExpression* ExprTransformer::TransformBool(
    const ExprState* es) {

  auto bool_expr = reinterpret_cast<const BoolExpr*>(es->expr);
  auto bool_state = reinterpret_cast<const BoolExprState*>(es);

  auto bool_op = bool_expr->boolop;

  /*
   * AND and OR can take >=2 arguments,
   * while NOT should take only one.
   */
  assert(bool_state->args);
  assert(bool_op != NOT_EXPR || list_length(bool_state->args) == 1);
  assert(bool_op == NOT_EXPR || list_length(bool_state->args) >= 2);

  auto args = bool_state->args;

  switch(bool_op){

    case AND_EXPR:
      LOG_INFO("Bool AND list \n");
      return TransformList(reinterpret_cast<const ExprState*>(args), EXPRESSION_TYPE_CONJUNCTION_AND);

    case OR_EXPR:
      LOG_INFO("Bool OR list \n");
      return TransformList(reinterpret_cast<const ExprState*>(args), EXPRESSION_TYPE_CONJUNCTION_OR);

    case NOT_EXPR:
    {
      LOG_INFO("Bool NOT \n");
      auto child_es = reinterpret_cast<const ExprState*>(lfirst(list_head(args)));
      auto child = TransformExpr(child_es);
      return expression::OperatorFactory(EXPRESSION_TYPE_OPERATOR_NOT, child, nullptr);
    }

    default:
      LOG_ERROR("Unrecognized BoolExpr : %u", bool_op);
  }

  return nullptr;
}

expression::AbstractExpression* ExprTransformer::TransformParam(const ExprState *es) {

  auto param_expr = reinterpret_cast<const Param*>(es->expr);

  switch (param_expr->paramkind) {
    case PARAM_EXTERN:
      LOG_INFO("Handle EXTREN PARAM");
      return expression::ParameterValueFactory(param_expr->paramid + 1); // 1 indexed
      break;
    default:
      LOG_ERROR("Unrecognized param kind %d", param_expr->paramkind);
      break;
  }

  return nullptr;
}

expression::AbstractExpression* ExprTransformer::TransformRelabelType(const ExprState *es) {
  auto state = reinterpret_cast<const GenericExprState*>(es);
  auto expr = reinterpret_cast<const RelabelType *>(es->expr);
  auto child_state = state->arg;
  auto child_expr = expr->arg;

  assert(expr->xpr.type == T_RelabelType);

  LOG_INFO("child is of type %d, %d", child_expr->type, child_state->type);
  LOG_INFO("%d, %d", expr->resulttype, expr->relabelformat);
  expression::AbstractExpression *child = ExprTransformer::TransformExpr(child_state);

  //TODO: not implemented yet

  return child;
}

expression::AbstractExpression* ExprTransformer::TransformFunc(const ExprState *es) {
  auto state = reinterpret_cast<const FuncExprState*>(es);
  auto expr = reinterpret_cast<const FuncExpr*>(es->expr);
  auto expr_args = reinterpret_cast<const ExprState*>(state->args);

  assert(expr->xpr.type == T_FuncExpr);

  LOG_INFO("Return type: %d, isReturn %d, Coercion: %d", expr->funcresulttype, expr->funcretset, expr->funcformat);
  expression::AbstractExpression *args = ExprTransformer::TransformExpr(expr_args);
  LOG_INFO("args : %s", args->DebugInfo(" ").c_str());

  //TODO: not implemented yet

  return nullptr;

}

expression::AbstractExpression*
ExprTransformer::TransformList(const ExprState* es, ExpressionType et) {

  assert(et == EXPRESSION_TYPE_CONJUNCTION_AND || et == EXPRESSION_TYPE_CONJUNCTION_OR);

  const List* list = reinterpret_cast<const List*>(es);
  ListCell *l;
  int length = list_length(list);
  assert(length > 0);
  LOG_INFO("Expression List of length %d", length);
  std::list<expression::AbstractExpression*> exprs; // a list of AND'ed expressions

  foreach(l, list)
  {
    const ExprState *expr_state = reinterpret_cast<const ExprState*>(lfirst(l));
    exprs.push_back(ExprTransformer::TransformExpr(expr_state));
  }

  return expression::ConjunctionFactory(et, exprs);

}

/**
 * @brief Helper function: re-map Postgres's builtin function Oid
 * to proper expression type in Peloton
 *
 * @param pg_func_id  PG Function Id used to lookup function in \b fmrg_builtin[]
 * (see Postgres source file 'fmgrtab.cpp')
 * @param lc          Left child.
 * @param rc          Right child.
 * @return            Corresponding expression tree in peloton.
 */
expression::AbstractExpression*
ReMapPgFunc(Oid func_id,
            expression::AbstractExpression* lc,
            expression::AbstractExpression* rc) {

  auto itr = pg_func_map.find(func_id);

  if(itr == pg_func_map.end()){
    LOG_ERROR("Unsupported PG Function ID : %u (check fmgrtab.cpp)\n", func_id);
    return nullptr;
  }

  auto pl_expr_type = itr->second;

  switch(pl_expr_type){

    case EXPRESSION_TYPE_COMPARE_EQ:
    case EXPRESSION_TYPE_COMPARE_NE:
    case EXPRESSION_TYPE_COMPARE_GT:
    case EXPRESSION_TYPE_COMPARE_LT:
    case EXPRESSION_TYPE_COMPARE_GTE:
    case EXPRESSION_TYPE_COMPARE_LTE:
      return expression::ComparisonFactory(pl_expr_type, lc, rc);

    case EXPRESSION_TYPE_OPERATOR_PLUS:
    case EXPRESSION_TYPE_OPERATOR_MINUS:
    case EXPRESSION_TYPE_OPERATOR_MULTIPLY:
    case EXPRESSION_TYPE_OPERATOR_DIVIDE:
      return expression::OperatorFactory(pl_expr_type, lc, rc);

    default:
      LOG_ERROR("This Peloton ExpressionType is in our map but not transformed here : %u", pl_expr_type);
  }

  return nullptr;
}

}  // namespace bridge
}  // namespace peloton
<|endoftext|>


<commit_before>#ifndef SHL_ARRAY_EXPANDABLE__H
#define SHL_ARRAY_EXPANDABLE__H

#include "shl.h"
#include "shl_timer.hpp"

#include <map>
#include <vector>
#include <cstring>

extern "C" {
#include <pthread.h>
}

#define SANITY_CHECK
#define WS_BUFFER_SIZE (1024*1024*100)

/**
 * \brief Array that is initally single-copy, but can be expanded to
 * use replication
 *
 * In expanded mode, replicas are not synchronized (i.e. writes are
 * local to current replica)
 *
 * Consistency:
 * - expanded mode  -> clients read their own writes, but not the others
 * - collapsed mode -> all clients read all writes (only one identical copy)
 *
 * I think we need a lock to switch modes (or a barrier). Wait until
 * all clients call collapse() and block until operation is finished.
 *
 * NOTE on concurrency:
 * - we assume that concurrent accesses only happen in collapsed mode
 * - we assume that collapse is only called AFTER the access model changed
 *   back to single-threaded
 *
 * In expanded mode, we need to keep track of writes. We do this using write
 * sets.
 *
 * We assume that the write sets from individual clients are
 * disjunct. If they are not, the semantics of concurrent writes in
 * the presence of parallel OMP loops is undeterministc anyway, so we
 * can just write back changes in any order we want (I think).
 *
 * Consistency: This should implement release consistency, I think:
 * collapse -> release, expand -> acquire.
 */

#ifdef SHL_DBG_ARRAY
#define debug_printf(x...) printf(x)
#else
#define debug_printf(x...) void()
#endif

union counter_element {
    int counter;
    char dummy[CACHELINE];
};

template <class T>
class shl_array_expandable : public shl_array_replicated<T>
{

public:
    Timer t_collapse;
    Timer t_expand[MAXCORES];

private:
    Timer t_write_back[MAXCORES];
    union counter_element c_write_back[MAXCORES];

    /**
     * \brief Check consistency of arrays
     *
     * Checks for each replica if it is consistency with the master copy
     */
    bool check_consistency(void)
    {
        for (int j=0; j<shl_array_replicated<T>::num_replicas; j++) {

            size_t num_total = shl_array<T>::size;
            for (size_t i=0; i<num_total; i++) {

                if (shl_array<T>::array[i] !=
                    shl_array_expandable<T>::rep_array[j][i]) {

                    printf("check_consistency failed for index %d\n", i);
                    printf("    ");

                    return false;
                }
            }
        }

        return true;
    }

    /**
     * \brief Write back write set of current thread to master copy
     */
    void ws_write_back(void)
    {
        // Nothing to do here
    }

// public:
//     // Make master array used distribution
//     virtual int get_options(void)
//     {
//         return shl_array<T>::get_options() | SHL_MALLOC_DISTRIBUTED;
//     }

protected:

    bool is_expanded[MAXCORES];
    pthread_barrier_t b;

public:
    /**
     * \brief Initialize replicated array
     */
    shl_array_expandable(size_t s, const char *_name, int (*f_lookup)(void))
        : shl_array_replicated<T>(s, _name, f_lookup)
    {
        printf("shl_array_expandable: setting %d threads\n", shl__num_threads());
        pthread_barrier_init(&b, NULL, shl__num_threads());
    }

    void expand(void)
    {
        // static size_t num_expand = 0;
        // num_expand++;

        int tid = shl__get_tid();

        //        t_expand[tid].start();

        //        assert(!is_expanded);

        //        int r = pthread_barrier_wait(&b);
        //        assert(r==PTHREAD_BARRIER_SERIAL_THREAD || r==0);

        // Every thread copies their own data
        if (shl__is_rep_coordinator(shl__get_tid())) {
            printf("Thread %d executes a memcpy for replica %d\n",
                   shl__get_tid(), shl__get_rep_id());
            memcpy((void*)shl_array_replicated<T>::rep_array[shl__get_rep_id()],
                    (void*)shl_array<T>::array,
                    shl_array<T>::size*sizeof(T));
        }
        //        r = pthread_barrier_wait(&b);

        //        t_expand[tid].stop();

        is_expanded[tid] = true;

        //        assert(is_expanded);
    }

    /**
     * \brief Collapse arrays
     *
     * Writes to replicas will be synchronized back to the master copy.
     *
     * If multiple replicas have been written at the same index i1
     * since the last expand, the value i1' written back to the master
     * copy is undeterministic.
     */
    void collapse(void)
    {
        //        assert (is_expanded);

        is_expanded[shl__get_tid()] = false;

        pthread_barrier_wait(&b);

        //        assert (!is_expanded);
    }

    /**
     * \brief Allocate both arrays and switch to collapsed mode
     */
    virtual void alloc(void)
    {
        shl_array<T>::alloc(); shl_array<T>::alloc_done = false;
        shl_array_replicated<T>::alloc();

        for (int i=0; i<MAXCORES; i++) {
            is_expanded[i] = false;
        }
    }

    /**
     * \brief Return pointer to beginning of replica.
     *
     * Writes to this pointer will result in undefined behavior. The
     * set-function has to be used to write arrays.
     */
    virtual T* get_array(void)
    {
        if (shl_array<T>::alloc_done) {
            return is_expanded[shl__get_tid()] ? shl_array_replicated<T>::get_array() :
                shl_array<T>::get_array();
        } else {
            return NULL;
        }
    }

    /**
     * \brief Read data item from array.
     *
     * If array is expanded, this will read the data item from the
     * local replica. If colapsed, read from master copy.
     */
    virtual T get(size_t i)
    {
        return is_expanded[shl__get_tid()] ? shl_array_replicated<T>::get(i) : shl_array<T>::get(i);
    }

    /**
     * \brief Write to array.
     *
     * Collapsed mode: write to master-copy
     *
     */
    virtual void set(size_t i, T v)
    {
        if (is_expanded[shl__get_tid()]) {

            debug_printf("Setting new idx=%zu old=%d new=%d on thread %d\n",
                         i, get(i), v, shl__get_tid());

            int rep_id = shl_array_replicated<T>::lookup();
            set_cached(i, v, (struct array_cache) { .rid = rep_id,
                        .tid = shl__get_tid() });
        }
        else {
            shl_array<T>::set(i, v);
        }
    }

    /**
     * \brief Write to the array
     *
     * Guarantees: threads read their own writes
     */
    void set_cached(size_t i, T v, struct array_cache c)
    {
        debug_printf("set_cached: %zu: %d -> %d on rep %d thread %d\n",
                     i, shl_array_replicated<T>::rep_array[c.rid][i], v,
                     c.rid, c.tid);

        shl_array_replicated<T>::rep_array[c.rid][i] = v;
        shl_array<T>::array[i] = v;
    }

    bool get_expanded(void)
    {
        return is_expanded[shl__get_tid()];
    }

    virtual ~shl_array_expandable(void)
    {
        pthread_barrier_destroy(&b);

#if defined(SHL_DBG_ARR)
        for (int i=0; i<shl__num_threads(); i++) {

            printf("destructor %2d: t_write_back %11.6lf - c_write_back % 4d\n",
                   i, t_write_back[i].get(), c_write_back[i].counter);
        }
#endif /* SHL_DBG_ARR */
    }

protected:
    virtual void print_options(void)
    {
        shl_array<T>::print_options();
        printf("expandable=[X]");
    }
};

#endif /* SHL_ARRAY_EXPANDABLE_H */
<commit_msg>e/c: for now, always copy back<commit_after>#ifndef SHL_ARRAY_EXPANDABLE__H
#define SHL_ARRAY_EXPANDABLE__H

#include "shl.h"
#include "shl_timer.hpp"

#include <map>
#include <vector>
#include <cstring>

extern "C" {
#include <pthread.h>
}

#define SANITY_CHECK
#define WS_BUFFER_SIZE (1024*1024*100)

/**
 * \brief Array that is initally single-copy, but can be expanded to
 * use replication
 *
 * In expanded mode, replicas are not synchronized (i.e. writes are
 * local to current replica)
 *
 * Consistency:
 * - expanded mode  -> clients read their own writes, but not the others
 * - collapsed mode -> all clients read all writes (only one identical copy)
 *
 * I think we need a lock to switch modes (or a barrier). Wait until
 * all clients call collapse() and block until operation is finished.
 *
 * NOTE on concurrency:
 * - we assume that concurrent accesses only happen in collapsed mode
 * - we assume that collapse is only called AFTER the access model changed
 *   back to single-threaded
 *
 * In expanded mode, we need to keep track of writes. We do this using write
 * sets.
 *
 * We assume that the write sets from individual clients are
 * disjunct. If they are not, the semantics of concurrent writes in
 * the presence of parallel OMP loops is undeterministc anyway, so we
 * can just write back changes in any order we want (I think).
 *
 * Consistency: This should implement release consistency, I think:
 * collapse -> release, expand -> acquire.
 */

#ifdef SHL_DBG_ARRAY
#define debug_printf(x...) printf(x)
#else
#define debug_printf(x...) void()
#endif

union counter_element {
    int counter;
    char dummy[CACHELINE];
};

template <class T>
class shl_array_expandable : public shl_array_replicated<T>
{

public:
    Timer t_collapse;
    Timer t_expand[MAXCORES];

private:
    Timer t_write_back[MAXCORES];
    union counter_element c_write_back[MAXCORES];

    /**
     * \brief Check consistency of arrays
     *
     * Checks for each replica if it is consistency with the master copy
     */
    bool check_consistency(void)
    {
        for (int j=0; j<shl_array_replicated<T>::num_replicas; j++) {

            size_t num_total = shl_array<T>::size;
            for (size_t i=0; i<num_total; i++) {

                if (shl_array<T>::array[i] !=
                    shl_array_expandable<T>::rep_array[j][i]) {

                    printf("check_consistency failed for index %d\n", i);
                    printf("    ");

                    return false;
                }
            }
        }

        return true;
    }

    /**
     * \brief Write back write set of current thread to master copy
     */
    void ws_write_back(void)
    {
        // Nothing to do here
    }

// public:
//     // Make master array used distribution
//     virtual int get_options(void)
//     {
//         return shl_array<T>::get_options() | SHL_MALLOC_DISTRIBUTED;
//     }

protected:

    bool is_expanded[MAXCORES];
    pthread_barrier_t b;

public:
    /**
     * \brief Initialize replicated array
     */
    shl_array_expandable(size_t s, const char *_name, int (*f_lookup)(void))
        : shl_array_replicated<T>(s, _name, f_lookup)
    {
        printf("shl_array_expandable: setting %d threads\n", shl__num_threads());
        pthread_barrier_init(&b, NULL, shl__num_threads());
    }

    void expand(void)
    {
        // static size_t num_expand = 0;
        // num_expand++;

        int tid = shl__get_tid();

        //        t_expand[tid].start();

        //        assert(!is_expanded);

        //        int r = pthread_barrier_wait(&b);
        //        assert(r==PTHREAD_BARRIER_SERIAL_THREAD || r==0);

        // Every thread copies their own data
        if (shl__is_rep_coordinator(shl__get_tid())) {
            printf("Thread %d executes a memcpy for replica %d\n",
                   shl__get_tid(), shl__get_rep_id());
            memcpy((void*)shl_array_replicated<T>::rep_array[shl__get_rep_id()],
                    (void*)shl_array<T>::array,
                    shl_array<T>::size*sizeof(T));
        }
        //        r = pthread_barrier_wait(&b);

        //        t_expand[tid].stop();

        is_expanded[tid] = true;

        //        assert(is_expanded);
    }

    /**
     * \brief Collapse arrays
     *
     * Writes to replicas will be synchronized back to the master copy.
     *
     * If multiple replicas have been written at the same index i1
     * since the last expand, the value i1' written back to the master
     * copy is undeterministic.
     */
    void collapse(void)
    {
        //        assert (is_expanded);

        is_expanded[shl__get_tid()] = false;

        pthread_barrier_wait(&b);

        //        assert (!is_expanded);
    }

    /**
     * \brief Allocate both arrays and switch to collapsed mode
     */
    virtual void alloc(void)
    {
        shl_array<T>::alloc(); shl_array<T>::alloc_done = false;
        shl_array_replicated<T>::alloc();

        for (int i=0; i<MAXCORES; i++) {
            is_expanded[i] = false;
        }
    }

    /**
     * \brief Return pointer to beginning of replica.
     *
     * Writes to this pointer will result in undefined behavior. The
     * set-function has to be used to write arrays.
     */
    virtual T* get_array(void)
    {
        if (shl_array<T>::alloc_done) {
            return is_expanded[shl__get_tid()] ? shl_array_replicated<T>::get_array() :
                shl_array<T>::get_array();
        } else {
            return NULL;
        }
    }

    /**
     * \brief Read data item from array.
     *
     * If array is expanded, this will read the data item from the
     * local replica. If colapsed, read from master copy.
     */
    virtual T get(size_t i)
    {
        return is_expanded[shl__get_tid()] ? shl_array_replicated<T>::get(i) : shl_array<T>::get(i);
    }

    /**
     * \brief Write to array.
     *
     * Collapsed mode: write to master-copy
     *
     */
    virtual void set(size_t i, T v)
    {
        if (is_expanded[shl__get_tid()]) {

            debug_printf("Setting new idx=%zu old=%d new=%d on thread %d\n",
                         i, get(i), v, shl__get_tid());

            int rep_id = shl_array_replicated<T>::lookup();
            set_cached(i, v, (struct array_cache) { .rid = rep_id,
                        .tid = shl__get_tid() });
        }
        else {
            shl_array<T>::set(i, v);
        }
    }

    /**
     * \brief Write to the array
     *
     * Guarantees: threads read their own writes
     */
    void set_cached(size_t i, T v, struct array_cache c)
    {
        debug_printf("set_cached: %zu: %d -> %d on rep %d thread %d\n",
                     i, shl_array_replicated<T>::rep_array[c.rid][i], v,
                     c.rid, c.tid);

        shl_array_replicated<T>::rep_array[c.rid][i] = v;
        shl_array<T>::array[i] = v;
    }

    bool get_expanded(void)
    {
        return is_expanded[shl__get_tid()];
    }

    virtual ~shl_array_expandable(void)
    {
        pthread_barrier_destroy(&b);

#if defined(SHL_DBG_ARR)
        for (int i=0; i<shl__num_threads(); i++) {

            printf("destructor %2d: t_write_back %11.6lf - c_write_back % 4d\n",
                   i, t_write_back[i].get(), c_write_back[i].counter);
        }
#endif /* SHL_DBG_ARR */
    }

protected:
    virtual void print_options(void)
    {
        shl_array<T>::print_options();
        printf("expandable=[X]");
    }

    virtual void copy_back(T* a)
    {
        shl_array<T>::copy_back(a);
    }

    virtual bool do_copy_back(void)
    {
        return true;
    }
};

#endif /* SHL_ARRAY_EXPANDABLE_H */
<|endoftext|>