<commit_before>/*
 Copyright 2015 Nervana Systems Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

#pragma once

#include <assert.h>

#include <vector>
#include <cstdio>
#include <iostream>
#include <chrono>
#include <utility>
#include <algorithm>

#include "threadpool.hpp"
#include "media.hpp"
#include "matrix.hpp"
#include "device.hpp"
#include "batch_iterator.hpp"
#include "manifest.hpp"
#include "provider_image_class.hpp"

/* DecodeThreadPool
 *
 * DecodeThreadPool takes data from the BufferPool `in`, transforms it
 * using `count` threads with a Media::transform built from
 * `mediaParams`.  Each minibatch is transposed by a manager thread.
 *
 */
class DecodeThreadPool : public ThreadPool {
public:
    DecodeThreadPool(int count, int batchSize,
                     int datumSize, int datumTypeSize,
                     int targetSize, int targetTypeSize,
                     const std::shared_ptr<BufferPool>& in, const std::shared_ptr<BufferPool>& out,
                     const std::shared_ptr<Device>& device,
                     nlohmann::json configJs);
                     // MediaParams* mediaParams);
    virtual ~DecodeThreadPool();
    virtual void start();
    virtual void stop();

protected:
    virtual void run(int id);
    virtual void work(int id);
    void produce();
    void consume();
    void manage();

private:
    DecodeThreadPool();
    DecodeThreadPool(const DecodeThreadPool&);

    int                         _itemsPerThread;
    std::shared_ptr<BufferPool> _in;
    std::shared_ptr<BufferPool> _out;
    std::mutex                  _mutex;
    std::condition_variable     _started;
    std::condition_variable     _ended;
    std::vector<int>            _startSignaled;
    int                         _endSignaled;
    std::thread*                _manager;
    bool                        _stopManager;
    bool                        _managerStopped;
    BufferPair*                 _inputBuf;
    int                         _bufferIndex;
    int                         _batchSize;
    std::vector<int>            _startInds;
    std::vector<int>            _endInds;
    std::vector<int>            _dataOffsets;
    std::vector<int>            _targetOffsets;
    int                         _datumSize;
    int                         _datumTypeSize;
    int                         _targetSize;
    int                         _targetTypeSize;
    // Datum length in bytes. (should we start using size_t for these?)
    int                         _datumLen;
    // Target length in bytes.
    int                         _targetLen;
    std::shared_ptr<Device>     _device;
    // std::vector<std::shared_ptr<Media>> _media;
    std::vector<std::shared_ptr<nervana::train_base>> _providers;
};

class ReadThread: public ThreadPool {
public:
    ReadThread(const std::shared_ptr<BufferPool>& out, const std::shared_ptr<BatchIterator>& batch_iterator);

protected:
    virtual void work(int id);

private:
    ReadThread();
    ReadThread(const ReadThread&);
    std::shared_ptr<BufferPool> _out;
    std::shared_ptr<BatchIterator> _batch_iterator;
};

/* Loader
 *
 * The job of the Loader is to copy data from BufferPair shared with
 * an ArchiveReader
 */
class Loader {
public:
    Loader(int miniBatchSize,
           bool shuffleManifest, bool shuffleEveryEpoch,
           int datumSize, int datumTypeSize,
           int targetSize, int targetTypeSize,
           int subsetPercent,
           const char* mediaConfigString,
           // MediaParams* mediaParams,
           DeviceParams* deviceParams,
           const char* manifestFilename,
           int macroBatchSize,
           const char* rootCacheDir,
           uint randomSeed);

    virtual ~Loader();
    int start();
    void stop();
    int reset();
    void next();
    std::shared_ptr<Device> getDevice();
    std::shared_ptr<BatchIterator> getBatchIterator();
    int itemCount();

private:
    void drain();

private:
    Loader();
    Loader(const Loader&);
    bool                                _first;
    int                                 _miniBatchSize;
    int                                 _datumSize;
    int                                 _datumTypeSize;
    int                                 _targetSize;
    int                                 _targetTypeSize;
    std::shared_ptr<BufferPool>         _readBufs;
    std::shared_ptr<BufferPool>         _decodeBufs;
    std::unique_ptr<ReadThread>         _readThread;
    std::unique_ptr<DecodeThreadPool>   _decodeThreads;
    std::shared_ptr<Device>             _device;
    std::shared_ptr<BatchIterator>      _batch_iterator;
    std::shared_ptr<Manifest>           _manifest;
    std::string                         _mediaConfigString;
    // MediaParams*                        _mediaParams;
};
<commit_msg>more comments in loader.cpp<commit_after>/*
 Copyright 2015 Nervana Systems Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

#pragma once

#include <assert.h>

#include <vector>
#include <cstdio>
#include <iostream>
#include <chrono>
#include <utility>
#include <algorithm>

#include "threadpool.hpp"
#include "media.hpp"
#include "matrix.hpp"
#include "device.hpp"
#include "batch_iterator.hpp"
#include "manifest.hpp"
#include "provider_image_class.hpp"

/* DecodeThreadPool
 *
 * DecodeThreadPool takes data from the BufferPool `in`, transforms it
 * using `count` threads with a Media::transform built from
 * `mediaParams`.  Each minibatch is transposed by a manager thread and
 * then copied to the `device`.
 *
 */
class DecodeThreadPool : public ThreadPool {
public:
    DecodeThreadPool(int count, int batchSize,
                     int datumSize, int datumTypeSize,
                     int targetSize, int targetTypeSize,
                     const std::shared_ptr<BufferPool>& in, const std::shared_ptr<BufferPool>& out,
                     const std::shared_ptr<Device>& device,
                     nlohmann::json configJs);
                     // MediaParams* mediaParams);
    virtual ~DecodeThreadPool();
    virtual void start();
    virtual void stop();

protected:
    virtual void run(int id);
    virtual void work(int id);
    void produce();
    void consume();
    void manage();

private:
    DecodeThreadPool();
    DecodeThreadPool(const DecodeThreadPool&);

    int                         _itemsPerThread;
    std::shared_ptr<BufferPool> _in;
    std::shared_ptr<BufferPool> _out;
    std::mutex                  _mutex;
    std::condition_variable     _started;
    std::condition_variable     _ended;
    std::vector<int>            _startSignaled;
    int                         _endSignaled;
    std::thread*                _manager;
    bool                        _stopManager;
    bool                        _managerStopped;
    BufferPair*                 _inputBuf;
    int                         _bufferIndex;
    int                         _batchSize;
    std::vector<int>            _startInds;
    std::vector<int>            _endInds;
    std::vector<int>            _dataOffsets;
    std::vector<int>            _targetOffsets;
    int                         _datumSize;
    int                         _datumTypeSize;
    int                         _targetSize;
    int                         _targetTypeSize;
    // Datum length in bytes. (should we start using size_t for these?)
    int                         _datumLen;
    // Target length in bytes.
    int                         _targetLen;
    std::shared_ptr<Device>     _device;
    // std::vector<std::shared_ptr<Media>> _media;
    std::vector<std::shared_ptr<nervana::train_base>> _providers;
};

/*
 * The ReadThread wraps BatchIterator in a thread an coordinates work
 * with other threads via locks on the output BufferPool `out`
 *
 */

class ReadThread: public ThreadPool {
public:
    ReadThread(const std::shared_ptr<BufferPool>& out, const std::shared_ptr<BatchIterator>& batch_iterator);

protected:
    virtual void work(int id);

private:
    ReadThread();
    ReadThread(const ReadThread&);
    std::shared_ptr<BufferPool> _out;
    std::shared_ptr<BatchIterator> _batch_iterator;
};

/* Loader
 *
 * The Loader instantiates and then coordinates the effort of
 * loading ingested data, caching blocks of it in contiguous
 * disk (using cpio file format), transforming the data and
 * finally loading the data into device memory
 *
 */
class Loader {
public:
    Loader(int miniBatchSize,
           bool shuffleManifest, bool shuffleEveryEpoch,
           int datumSize, int datumTypeSize,
           int targetSize, int targetTypeSize,
           int subsetPercent,
           const char* mediaConfigString,
           // MediaParams* mediaParams,
           DeviceParams* deviceParams,
           const char* manifestFilename,
           int macroBatchSize,
           const char* rootCacheDir,
           uint randomSeed);

    virtual ~Loader();
    int start();
    void stop();
    int reset();
    void next();
    std::shared_ptr<Device> getDevice();
    std::shared_ptr<BatchIterator> getBatchIterator();
    int itemCount();

private:
    void drain();

private:
    Loader();
    Loader(const Loader&);
    bool                                _first;
    int                                 _miniBatchSize;
    int                                 _datumSize;
    int                                 _datumTypeSize;
    int                                 _targetSize;
    int                                 _targetTypeSize;
    std::shared_ptr<BufferPool>         _readBufs;
    std::shared_ptr<BufferPool>         _decodeBufs;
    std::unique_ptr<ReadThread>         _readThread;
    std::unique_ptr<DecodeThreadPool>   _decodeThreads;
    std::shared_ptr<Device>             _device;
    std::shared_ptr<BatchIterator>      _batch_iterator;
    std::shared_ptr<Manifest>           _manifest;
    std::string                         _mediaConfigString;
    // MediaParams*                        _mediaParams;
};
<|endoftext|>


<commit_before>// Scintilla source code edit control
/** @file LexScriptol.cxx
 ** Lexer for Scriptol.
 **/

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <stdarg.h>
#include <assert.h>
#include <ctype.h>

#include "ILexer.h"
#include "Scintilla.h"
#include "SciLexer.h"

#include "WordList.h"
#include "LexAccessor.h"
#include "Accessor.h"
#include "StyleContext.h"
#include "CharacterSet.h"
#include "LexerModule.h"

#ifdef SCI_NAMESPACE
using namespace Scintilla;
#endif

static void ClassifyWordSol(unsigned int start, unsigned int end, WordList &keywords, Accessor &styler, char *prevWord)
{
    char s[100];
    bool wordIsNumber = isdigit(styler[start]) != 0;
    for (unsigned int i = 0; i < end - start + 1 && i < 30; i++)
     {
           s[i] = styler[start + i];
           s[i + 1] = '\0';
     }
    char chAttr = SCE_SCRIPTOL_IDENTIFIER;
    if (0 == strcmp(prevWord, "class"))       chAttr = SCE_SCRIPTOL_CLASSNAME;
    else if (wordIsNumber)                    chAttr = SCE_SCRIPTOL_NUMBER;
    else if (keywords.InList(s))              chAttr = SCE_SCRIPTOL_KEYWORD;
    else for (unsigned int i = 0; i < end - start + 1; i++)  // test dotted idents
    {
        if (styler[start + i] == '.')
        {
            styler.ColourTo(start + i - 1, chAttr);
            styler.ColourTo(start + i, SCE_SCRIPTOL_OPERATOR);
        }
    }
    styler.ColourTo(end, chAttr);
    strcpy(prevWord, s);
}

static bool IsSolComment(Accessor &styler, int pos, int len)
{
   if(len > 0)
   {
     char c = styler[pos];
     if(c == '`') return true;
     if(len > 1)
     {
        if(c == '/')
        {
          c = styler[pos + 1];
          if(c == '/') return true;
          if(c == '*') return true;
        }
     }
   }
   return false;
}

static bool IsSolStringStart(char ch)
{
    if (ch == '\'' || ch == '"')  return true;
    return false;
}

static bool IsSolWordStart(char ch)
{
    return (iswordchar(ch) && !IsSolStringStart(ch));
}

static int GetSolStringState(Accessor &styler, int i, int *nextIndex)
{
	char ch = styler.SafeGetCharAt(i);
	char chNext = styler.SafeGetCharAt(i + 1);

        if (ch != '\"' && ch != '\'')
        {
            *nextIndex = i + 1;
            return SCE_SCRIPTOL_DEFAULT;
	}
        // ch is either single or double quotes in string
        // code below seem non-sense but is here for future extensions
	if (ch == chNext && ch == styler.SafeGetCharAt(i + 2))
        {
          *nextIndex = i + 3;
          if(ch == '\"') return SCE_SCRIPTOL_TRIPLE;
          if(ch == '\'') return SCE_SCRIPTOL_TRIPLE;
          return SCE_SCRIPTOL_STRING;
	}
        else
        {
          *nextIndex = i + 1;
          if (ch == '"') return SCE_SCRIPTOL_STRING;
          else           return SCE_SCRIPTOL_STRING;
	}
}

static void ColouriseSolDoc(unsigned int startPos, int length, int initStyle,
                            WordList *keywordlists[], Accessor &styler)
 {

	int lengthDoc = startPos + length;
        char stringType = '\"';

	if (startPos > 0)
        {
            int lineCurrent = styler.GetLine(startPos);
            if (lineCurrent > 0)
            {
              startPos = styler.LineStart(lineCurrent-1);
              if (startPos == 0) initStyle = SCE_SCRIPTOL_DEFAULT;
              else               initStyle = styler.StyleAt(startPos-1);
            }
	}

	styler.StartAt(startPos, 127);

	WordList &keywords = *keywordlists[0];

	int whingeLevel = styler.GetPropertyInt("tab.timmy.whinge.level");
	char prevWord[200];
	prevWord[0] = '\0';
        if (length == 0)  return;

	int state = initStyle & 31;

	int nextIndex = 0;
        char chPrev  = ' ';
        char chPrev2 = ' ';
        char chNext  = styler[startPos];
	styler.StartSegment(startPos);
	bool atStartLine = true;
	int spaceFlags = 0;
	for (int i = startPos; i < lengthDoc; i++)
        {

         if (atStartLine)
         {
         char chBad = static_cast<char>(64);
         char chGood = static_cast<char>(0);
         char chFlags = chGood;

         if (whingeLevel == 1)
         {
             chFlags = (spaceFlags & wsInconsistent) ? chBad : chGood;
         }
         else if (whingeLevel == 2)
         {
             chFlags = (spaceFlags & wsSpaceTab) ? chBad : chGood;
         }
         else if (whingeLevel == 3)
         {
             chFlags = (spaceFlags & wsSpace) ? chBad : chGood;
         }
         else if (whingeLevel == 4)
         {
              chFlags = (spaceFlags & wsTab) ? chBad : chGood;
         }
         styler.SetFlags(chFlags, static_cast<char>(state));
         atStartLine = false;
       }

       char ch = chNext;
       chNext = styler.SafeGetCharAt(i + 1);

       if ((ch == '\r' && chNext != '\n') || (ch == '\n') || (i == lengthDoc))
       {
          if ((state == SCE_SCRIPTOL_DEFAULT) ||
              (state == SCE_SCRIPTOL_TRIPLE) ||
              (state == SCE_SCRIPTOL_COMMENTBLOCK))
          {
              styler.ColourTo(i, state);
          }
          atStartLine = true;
        }

        if (styler.IsLeadByte(ch))
         {
             chNext = styler.SafeGetCharAt(i + 2);
             chPrev  = ' ';
             chPrev2 = ' ';
             i += 1;
             continue;
         }

        if (state == SCE_SCRIPTOL_STRINGEOL)
         {
             if (ch != '\r' && ch != '\n')
             {
                    styler.ColourTo(i - 1, state);
                    state = SCE_SCRIPTOL_DEFAULT;
             }
         }

        if (state == SCE_SCRIPTOL_DEFAULT)
         {
            if (IsSolWordStart(ch))
            {
                 styler.ColourTo(i - 1, state);
                 state = SCE_SCRIPTOL_KEYWORD;
            }
            else if (ch == '`')
            {
                styler.ColourTo(i - 1, state);
                state = SCE_SCRIPTOL_COMMENTLINE;
            }
            else if (ch == '/')
            {
                styler.ColourTo(i - 1, state);
                if(chNext == '/') state = SCE_SCRIPTOL_CSTYLE;
                if(chNext == '*') state = SCE_SCRIPTOL_COMMENTBLOCK;
            }

            else if (IsSolStringStart(ch))
            {
               styler.ColourTo(i - 1, state);
               state = GetSolStringState(styler, i, &nextIndex);
               if(state == SCE_SCRIPTOL_STRING)
               {
                 stringType = ch;
               }
               if (nextIndex != i + 1)
               {
                   i = nextIndex - 1;
                   ch = ' ';
                   chPrev = ' ';
                   chNext = styler.SafeGetCharAt(i + 1);
               }
           }
            else if (isoperator(ch))
            {
                 styler.ColourTo(i - 1, state);
                 styler.ColourTo(i, SCE_SCRIPTOL_OPERATOR);
            }
          }
          else if (state == SCE_SCRIPTOL_KEYWORD)
          {
              if (!iswordchar(ch))
              {
                 ClassifyWordSol(styler.GetStartSegment(), i - 1, keywords, styler, prevWord);
                 state = SCE_SCRIPTOL_DEFAULT;
                 if (ch == '`')
                 {
                     state = chNext == '`' ? SCE_SCRIPTOL_PERSISTENT : SCE_SCRIPTOL_COMMENTLINE;
                 }
                 else if (IsSolStringStart(ch))
                 {
                    styler.ColourTo(i - 1, state);
                    state = GetSolStringState(styler, i, &nextIndex);
                    if (nextIndex != i + 1)
                    {
                       i = nextIndex - 1;
                       ch = ' ';
                       chPrev = ' ';
                       chNext = styler.SafeGetCharAt(i + 1);
                     }
                 }
                 else if (isoperator(ch))
                 {
                     styler.ColourTo(i, SCE_SCRIPTOL_OPERATOR);
                 }
             }
          }
          else
          {
            if (state == SCE_SCRIPTOL_COMMENTLINE ||
                state == SCE_SCRIPTOL_PERSISTENT ||
                state == SCE_SCRIPTOL_CSTYLE)
            {
                 if (ch == '\r' || ch == '\n')
                 {
                     styler.ColourTo(i - 1, state);
                     state = SCE_SCRIPTOL_DEFAULT;
                 }
            }
            else if(state == SCE_SCRIPTOL_COMMENTBLOCK)
            {
              if(chPrev == '*' && ch == '/')
              {
                styler.ColourTo(i, state);
                state = SCE_SCRIPTOL_DEFAULT;
              }
            }
            else if ((state == SCE_SCRIPTOL_STRING) ||
                     (state == SCE_SCRIPTOL_CHARACTER))
            {
               if ((ch == '\r' || ch == '\n') && (chPrev != '\\'))
                {
                    styler.ColourTo(i - 1, state);
                    state = SCE_SCRIPTOL_STRINGEOL;
                }
                else if (ch == '\\')
                {
                   if (chNext == '\"' || chNext == '\'' || chNext == '\\')
                   {
                        i++;
                        ch = chNext;
                        chNext = styler.SafeGetCharAt(i + 1);
                   }
                 }
                else if ((ch == '\"') || (ch == '\''))
                {
                    // must match the entered quote type
                    if(ch == stringType)
                    {
                      styler.ColourTo(i, state);
                      state = SCE_SCRIPTOL_DEFAULT;
                    }
                 }
             }
             else if (state == SCE_SCRIPTOL_TRIPLE)
             {
                if ((ch == '\'' && chPrev == '\'' && chPrev2 == '\'') ||
                    (ch == '\"' && chPrev == '\"' && chPrev2 == '\"'))
                 {
                    styler.ColourTo(i, state);
                    state = SCE_SCRIPTOL_DEFAULT;
                 }
             }

           }
          chPrev2 = chPrev;
          chPrev = ch;
	}
        if (state == SCE_SCRIPTOL_KEYWORD)
        {
            ClassifyWordSol(styler.GetStartSegment(),
                 lengthDoc-1, keywords, styler, prevWord);
	}
        else
        {
            styler.ColourTo(lengthDoc-1, state);
	}
}

static void FoldSolDoc(unsigned int startPos, int length, int initStyle,
						   WordList *[], Accessor &styler)
 {
	int lengthDoc = startPos + length;

	int lineCurrent = styler.GetLine(startPos);
	if (startPos > 0)
        {
          if (lineCurrent > 0)
          {
               lineCurrent--;
               startPos = styler.LineStart(lineCurrent);
               if (startPos == 0)
                    initStyle = SCE_SCRIPTOL_DEFAULT;
               else
                    initStyle = styler.StyleAt(startPos-1);
           }
	}
	int state = initStyle & 31;
	int spaceFlags = 0;
        int indentCurrent = styler.IndentAmount(lineCurrent, &spaceFlags, IsSolComment);
        if ((state == SCE_SCRIPTOL_TRIPLE))
             indentCurrent |= SC_FOLDLEVELWHITEFLAG;
	char chNext = styler[startPos];
	for (int i = startPos; i < lengthDoc; i++)
         {
		char ch = chNext;
		chNext = styler.SafeGetCharAt(i + 1);
		int style = styler.StyleAt(i) & 31;

		if ((ch == '\r' && chNext != '\n') || (ch == '\n') || (i == lengthDoc))
                {
                   int lev = indentCurrent;
                   int indentNext = styler.IndentAmount(lineCurrent + 1, &spaceFlags, IsSolComment);
                   if (style == SCE_SCRIPTOL_TRIPLE)
                        indentNext |= SC_FOLDLEVELWHITEFLAG;
                   if (!(indentCurrent & SC_FOLDLEVELWHITEFLAG))
                    {
                        // Only non whitespace lines can be headers
                        if ((indentCurrent & SC_FOLDLEVELNUMBERMASK) < (indentNext & SC_FOLDLEVELNUMBERMASK))
                        {
                              lev |= SC_FOLDLEVELHEADERFLAG;
                        }
                        else if (indentNext & SC_FOLDLEVELWHITEFLAG)
                        {
                             // Line after is blank so check the next - maybe should continue further?
                             int spaceFlags2 = 0;
                             int indentNext2 = styler.IndentAmount(lineCurrent + 2, &spaceFlags2, IsSolComment);
                             if ((indentCurrent & SC_FOLDLEVELNUMBERMASK) < (indentNext2 & SC_FOLDLEVELNUMBERMASK))
                             {
                                   lev |= SC_FOLDLEVELHEADERFLAG;
                              }
                        }
                    }
                   indentCurrent = indentNext;
                   styler.SetLevel(lineCurrent, lev);
                   lineCurrent++;
		}
	}
}

LexerModule lmScriptol(SCLEX_SCRIPTOL, ColouriseSolDoc, "scriptol", FoldSolDoc);
<commit_msg>Fix warning from Clang.<commit_after>// Scintilla source code edit control
/** @file LexScriptol.cxx
 ** Lexer for Scriptol.
 **/

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <stdarg.h>
#include <assert.h>
#include <ctype.h>

#include "ILexer.h"
#include "Scintilla.h"
#include "SciLexer.h"

#include "WordList.h"
#include "LexAccessor.h"
#include "Accessor.h"
#include "StyleContext.h"
#include "CharacterSet.h"
#include "LexerModule.h"

#ifdef SCI_NAMESPACE
using namespace Scintilla;
#endif

static void ClassifyWordSol(unsigned int start, unsigned int end, WordList &keywords, Accessor &styler, char *prevWord)
{
    char s[100];
    bool wordIsNumber = isdigit(styler[start]) != 0;
    for (unsigned int i = 0; i < end - start + 1 && i < 30; i++)
     {
           s[i] = styler[start + i];
           s[i + 1] = '\0';
     }
    char chAttr = SCE_SCRIPTOL_IDENTIFIER;
    if (0 == strcmp(prevWord, "class"))       chAttr = SCE_SCRIPTOL_CLASSNAME;
    else if (wordIsNumber)                    chAttr = SCE_SCRIPTOL_NUMBER;
    else if (keywords.InList(s))              chAttr = SCE_SCRIPTOL_KEYWORD;
    else for (unsigned int i = 0; i < end - start + 1; i++)  // test dotted idents
    {
        if (styler[start + i] == '.')
        {
            styler.ColourTo(start + i - 1, chAttr);
            styler.ColourTo(start + i, SCE_SCRIPTOL_OPERATOR);
        }
    }
    styler.ColourTo(end, chAttr);
    strcpy(prevWord, s);
}

static bool IsSolComment(Accessor &styler, int pos, int len)
{
   if(len > 0)
   {
     char c = styler[pos];
     if(c == '`') return true;
     if(len > 1)
     {
        if(c == '/')
        {
          c = styler[pos + 1];
          if(c == '/') return true;
          if(c == '*') return true;
        }
     }
   }
   return false;
}

static bool IsSolStringStart(char ch)
{
    if (ch == '\'' || ch == '"')  return true;
    return false;
}

static bool IsSolWordStart(char ch)
{
    return (iswordchar(ch) && !IsSolStringStart(ch));
}

static int GetSolStringState(Accessor &styler, int i, int *nextIndex)
{
	char ch = styler.SafeGetCharAt(i);
	char chNext = styler.SafeGetCharAt(i + 1);

        if (ch != '\"' && ch != '\'')
        {
            *nextIndex = i + 1;
            return SCE_SCRIPTOL_DEFAULT;
	}
        // ch is either single or double quotes in string
        // code below seem non-sense but is here for future extensions
	if (ch == chNext && ch == styler.SafeGetCharAt(i + 2))
        {
          *nextIndex = i + 3;
          if(ch == '\"') return SCE_SCRIPTOL_TRIPLE;
          if(ch == '\'') return SCE_SCRIPTOL_TRIPLE;
          return SCE_SCRIPTOL_STRING;
	}
        else
        {
          *nextIndex = i + 1;
          if (ch == '"') return SCE_SCRIPTOL_STRING;
          else           return SCE_SCRIPTOL_STRING;
	}
}

static void ColouriseSolDoc(unsigned int startPos, int length, int initStyle,
                            WordList *keywordlists[], Accessor &styler)
 {

	int lengthDoc = startPos + length;
        char stringType = '\"';

	if (startPos > 0)
        {
            int lineCurrent = styler.GetLine(startPos);
            if (lineCurrent > 0)
            {
              startPos = styler.LineStart(lineCurrent-1);
              if (startPos == 0) initStyle = SCE_SCRIPTOL_DEFAULT;
              else               initStyle = styler.StyleAt(startPos-1);
            }
	}

	styler.StartAt(startPos, 127);

	WordList &keywords = *keywordlists[0];

	int whingeLevel = styler.GetPropertyInt("tab.timmy.whinge.level");
	char prevWord[200];
	prevWord[0] = '\0';
        if (length == 0)  return;

	int state = initStyle & 31;

	int nextIndex = 0;
        char chPrev  = ' ';
        char chPrev2 = ' ';
        char chNext  = styler[startPos];
	styler.StartSegment(startPos);
	bool atStartLine = true;
	int spaceFlags = 0;
	for (int i = startPos; i < lengthDoc; i++)
        {

         if (atStartLine)
         {
         char chBad = static_cast<char>(64);
         char chGood = static_cast<char>(0);
         char chFlags = chGood;

         if (whingeLevel == 1)
         {
             chFlags = (spaceFlags & wsInconsistent) ? chBad : chGood;
         }
         else if (whingeLevel == 2)
         {
             chFlags = (spaceFlags & wsSpaceTab) ? chBad : chGood;
         }
         else if (whingeLevel == 3)
         {
             chFlags = (spaceFlags & wsSpace) ? chBad : chGood;
         }
         else if (whingeLevel == 4)
         {
              chFlags = (spaceFlags & wsTab) ? chBad : chGood;
         }
         styler.SetFlags(chFlags, static_cast<char>(state));
         atStartLine = false;
       }

       char ch = chNext;
       chNext = styler.SafeGetCharAt(i + 1);

       if ((ch == '\r' && chNext != '\n') || (ch == '\n') || (i == lengthDoc))
       {
          if ((state == SCE_SCRIPTOL_DEFAULT) ||
              (state == SCE_SCRIPTOL_TRIPLE) ||
              (state == SCE_SCRIPTOL_COMMENTBLOCK))
          {
              styler.ColourTo(i, state);
          }
          atStartLine = true;
        }

        if (styler.IsLeadByte(ch))
         {
             chNext = styler.SafeGetCharAt(i + 2);
             chPrev  = ' ';
             chPrev2 = ' ';
             i += 1;
             continue;
         }

        if (state == SCE_SCRIPTOL_STRINGEOL)
         {
             if (ch != '\r' && ch != '\n')
             {
                    styler.ColourTo(i - 1, state);
                    state = SCE_SCRIPTOL_DEFAULT;
             }
         }

        if (state == SCE_SCRIPTOL_DEFAULT)
         {
            if (IsSolWordStart(ch))
            {
                 styler.ColourTo(i - 1, state);
                 state = SCE_SCRIPTOL_KEYWORD;
            }
            else if (ch == '`')
            {
                styler.ColourTo(i - 1, state);
                state = SCE_SCRIPTOL_COMMENTLINE;
            }
            else if (ch == '/')
            {
                styler.ColourTo(i - 1, state);
                if(chNext == '/') state = SCE_SCRIPTOL_CSTYLE;
                if(chNext == '*') state = SCE_SCRIPTOL_COMMENTBLOCK;
            }

            else if (IsSolStringStart(ch))
            {
               styler.ColourTo(i - 1, state);
               state = GetSolStringState(styler, i, &nextIndex);
               if(state == SCE_SCRIPTOL_STRING)
               {
                 stringType = ch;
               }
               if (nextIndex != i + 1)
               {
                   i = nextIndex - 1;
                   ch = ' ';
                   chPrev = ' ';
                   chNext = styler.SafeGetCharAt(i + 1);
               }
           }
            else if (isoperator(ch))
            {
                 styler.ColourTo(i - 1, state);
                 styler.ColourTo(i, SCE_SCRIPTOL_OPERATOR);
            }
          }
          else if (state == SCE_SCRIPTOL_KEYWORD)
          {
              if (!iswordchar(ch))
              {
                 ClassifyWordSol(styler.GetStartSegment(), i - 1, keywords, styler, prevWord);
                 state = SCE_SCRIPTOL_DEFAULT;
                 if (ch == '`')
                 {
                     state = chNext == '`' ? SCE_SCRIPTOL_PERSISTENT : SCE_SCRIPTOL_COMMENTLINE;
                 }
                 else if (IsSolStringStart(ch))
                 {
                    styler.ColourTo(i - 1, state);
                    state = GetSolStringState(styler, i, &nextIndex);
                    if (nextIndex != i + 1)
                    {
                       i = nextIndex - 1;
                       ch = ' ';
                       chPrev = ' ';
                       chNext = styler.SafeGetCharAt(i + 1);
                     }
                 }
                 else if (isoperator(ch))
                 {
                     styler.ColourTo(i, SCE_SCRIPTOL_OPERATOR);
                 }
             }
          }
          else
          {
            if (state == SCE_SCRIPTOL_COMMENTLINE ||
                state == SCE_SCRIPTOL_PERSISTENT ||
                state == SCE_SCRIPTOL_CSTYLE)
            {
                 if (ch == '\r' || ch == '\n')
                 {
                     styler.ColourTo(i - 1, state);
                     state = SCE_SCRIPTOL_DEFAULT;
                 }
            }
            else if(state == SCE_SCRIPTOL_COMMENTBLOCK)
            {
              if(chPrev == '*' && ch == '/')
              {
                styler.ColourTo(i, state);
                state = SCE_SCRIPTOL_DEFAULT;
              }
            }
            else if ((state == SCE_SCRIPTOL_STRING) ||
                     (state == SCE_SCRIPTOL_CHARACTER))
            {
               if ((ch == '\r' || ch == '\n') && (chPrev != '\\'))
                {
                    styler.ColourTo(i - 1, state);
                    state = SCE_SCRIPTOL_STRINGEOL;
                }
                else if (ch == '\\')
                {
                   if (chNext == '\"' || chNext == '\'' || chNext == '\\')
                   {
                        i++;
                        ch = chNext;
                        chNext = styler.SafeGetCharAt(i + 1);
                   }
                 }
                else if ((ch == '\"') || (ch == '\''))
                {
                    // must match the entered quote type
                    if(ch == stringType)
                    {
                      styler.ColourTo(i, state);
                      state = SCE_SCRIPTOL_DEFAULT;
                    }
                 }
             }
             else if (state == SCE_SCRIPTOL_TRIPLE)
             {
                if ((ch == '\'' && chPrev == '\'' && chPrev2 == '\'') ||
                    (ch == '\"' && chPrev == '\"' && chPrev2 == '\"'))
                 {
                    styler.ColourTo(i, state);
                    state = SCE_SCRIPTOL_DEFAULT;
                 }
             }

           }
          chPrev2 = chPrev;
          chPrev = ch;
	}
        if (state == SCE_SCRIPTOL_KEYWORD)
        {
            ClassifyWordSol(styler.GetStartSegment(),
                 lengthDoc-1, keywords, styler, prevWord);
	}
        else
        {
            styler.ColourTo(lengthDoc-1, state);
	}
}

static void FoldSolDoc(unsigned int startPos, int length, int initStyle,
						   WordList *[], Accessor &styler)
 {
	int lengthDoc = startPos + length;

	int lineCurrent = styler.GetLine(startPos);
	if (startPos > 0)
        {
          if (lineCurrent > 0)
          {
               lineCurrent--;
               startPos = styler.LineStart(lineCurrent);
               if (startPos == 0)
                    initStyle = SCE_SCRIPTOL_DEFAULT;
               else
                    initStyle = styler.StyleAt(startPos-1);
           }
	}
	int state = initStyle & 31;
	int spaceFlags = 0;
        int indentCurrent = styler.IndentAmount(lineCurrent, &spaceFlags, IsSolComment);
        if (state == SCE_SCRIPTOL_TRIPLE)
             indentCurrent |= SC_FOLDLEVELWHITEFLAG;
	char chNext = styler[startPos];
	for (int i = startPos; i < lengthDoc; i++)
         {
		char ch = chNext;
		chNext = styler.SafeGetCharAt(i + 1);
		int style = styler.StyleAt(i) & 31;

		if ((ch == '\r' && chNext != '\n') || (ch == '\n') || (i == lengthDoc))
                {
                   int lev = indentCurrent;
                   int indentNext = styler.IndentAmount(lineCurrent + 1, &spaceFlags, IsSolComment);
                   if (style == SCE_SCRIPTOL_TRIPLE)
                        indentNext |= SC_FOLDLEVELWHITEFLAG;
                   if (!(indentCurrent & SC_FOLDLEVELWHITEFLAG))
                    {
                        // Only non whitespace lines can be headers
                        if ((indentCurrent & SC_FOLDLEVELNUMBERMASK) < (indentNext & SC_FOLDLEVELNUMBERMASK))
                        {
                              lev |= SC_FOLDLEVELHEADERFLAG;
                        }
                        else if (indentNext & SC_FOLDLEVELWHITEFLAG)
                        {
                             // Line after is blank so check the next - maybe should continue further?
                             int spaceFlags2 = 0;
                             int indentNext2 = styler.IndentAmount(lineCurrent + 2, &spaceFlags2, IsSolComment);
                             if ((indentCurrent & SC_FOLDLEVELNUMBERMASK) < (indentNext2 & SC_FOLDLEVELNUMBERMASK))
                             {
                                   lev |= SC_FOLDLEVELHEADERFLAG;
                              }
                        }
                    }
                   indentCurrent = indentNext;
                   styler.SetLevel(lineCurrent, lev);
                   lineCurrent++;
		}
	}
}

LexerModule lmScriptol(SCLEX_SCRIPTOL, ColouriseSolDoc, "scriptol", FoldSolDoc);
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "iree/hal/llvmjit/llvmjit_executable.h"

#include <iostream>
#include <memory>

#include "flatbuffers/flatbuffers.h"
#include "iree/hal/executable.h"
#include "iree/schemas/llvmir_executable_def_generated.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/AsmParser/Parser.h"
#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
#include "llvm/ExecutionEngine/Orc/LLJIT.h"
#include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/SourceMgr.h"

namespace iree {
namespace hal {
namespace llvmjit {

// static
StatusOr<ref_ptr<LLVMJITExecutable>> LLVMJITExecutable::Load(
    hal::Allocator* allocator, ExecutableSpec spec,
    llvm::orc::LLJIT* execution_engine, bool allow_aliasing_data) {
  auto module_def =
      ::flatbuffers::GetRoot<LLVMIRExecutableDef>(spec.executable_data.data());
  auto data =
      reinterpret_cast<const char*>(module_def->llvmir_module()->data());
  const int size = module_def->llvmir_module()->size();
  auto mem_buffer = llvm::MemoryBuffer::getMemBufferCopy(
      llvm::StringRef(data, size), "llvm-ir");
  auto llvm_context = std::make_unique<llvm::LLVMContext>();
  llvm::SMDiagnostic sm_diagnostic;
  auto module = llvm::parseAssembly(*mem_buffer, sm_diagnostic, *llvm_context);
  if (!module)
    return InvalidArgumentErrorBuilder(IREE_LOC) << "Can't parse LLVMIR Module";
  auto dataLayout = module->getDataLayout();
  const auto entry_points = module_def->entry_points();
  llvm::orc::ThreadSafeModule thread_safe_module(std::move(module),
                                                 std::move(llvm_context));
  llvm::Error err =
      execution_engine->addIRModule(std::move(thread_safe_module));
  if (err)
    return InvalidArgumentErrorBuilder(IREE_LOC)
           << "Can't add executable module to execution engine";

  auto dylib_serarch_generator =
      llvm::orc::DynamicLibrarySearchGenerator::GetForCurrentProcess(
          dataLayout.getGlobalPrefix());
  if (!dylib_serarch_generator)
    return UnavailableErrorBuilder(IREE_LOC)
           << "Can't resolve symbols in current process";

  auto& main_jitdylib = execution_engine->getMainJITDylib();
  main_jitdylib.addGenerator(std::move(dylib_serarch_generator.get()));

  auto executable =
      make_ref<LLVMJITExecutable>(allocator, spec, allow_aliasing_data);

  for (const auto func_name : *entry_points) {
    auto func_symbol = execution_engine->lookup("invoke_" + func_name->str());
    if (!func_symbol) {
      return NotFoundErrorBuilder(IREE_LOC)
             << "Can't JIT compile function : " << func_name;
    }
    // Map function to its invoke_ symbol.
    executable->InsertSymbol(func_symbol.get());
  }

  return executable;
}

void LLVMJITExecutable::InsertSymbol(llvm::JITEvaluatedSymbol symbol) {
  symbols_.push_back(symbol);
}

Status LLVMJITExecutable::Invoke(int func_id,
                                 llvm::MutableArrayRef<void*> args) {
  const auto func_symbol = symbols_[func_id];
  auto exe_func = (void (*)(void**))func_symbol.getAddress();
  exe_func(args.data());
  return OkStatus();
}

LLVMJITExecutable::LLVMJITExecutable(hal::Allocator* allocator,
                                     ExecutableSpec spec,
                                     bool allow_aliasing_data)
    : spec_(spec) {
  if (!allow_aliasing_data) {
    // Clone data.
    cloned_executable_data_ = {spec.executable_data.begin(),
                               spec.executable_data.end()};
    spec_.executable_data = absl::MakeConstSpan(cloned_executable_data_);
  }
}

LLVMJITExecutable::~LLVMJITExecutable() = default;

}  // namespace llvmjit
}  // namespace hal
}  // namespace iree
<commit_msg>Emit better error message when compile/load llvmjit executable modules.<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "iree/hal/llvmjit/llvmjit_executable.h"

#include <iostream>
#include <memory>

#include "flatbuffers/flatbuffers.h"
#include "iree/hal/executable.h"
#include "iree/schemas/llvmir_executable_def_generated.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/AsmParser/Parser.h"
#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
#include "llvm/ExecutionEngine/Orc/LLJIT.h"
#include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/SourceMgr.h"

namespace iree {
namespace hal {
namespace llvmjit {

// static
StatusOr<ref_ptr<LLVMJITExecutable>> LLVMJITExecutable::Load(
    hal::Allocator* allocator, ExecutableSpec spec,
    llvm::orc::LLJIT* execution_engine, bool allow_aliasing_data) {
  auto module_def =
      ::flatbuffers::GetRoot<LLVMIRExecutableDef>(spec.executable_data.data());
  auto data =
      reinterpret_cast<const char*>(module_def->llvmir_module()->data());
  const int size = module_def->llvmir_module()->size();
  auto mem_buffer = llvm::MemoryBuffer::getMemBufferCopy(
      llvm::StringRef(data, size), "llvm-ir");
  auto llvm_context = std::make_unique<llvm::LLVMContext>();
  llvm::SMDiagnostic sm_diagnostic;
  auto module = llvm::parseAssembly(*mem_buffer, sm_diagnostic, *llvm_context);
  if (!module)
    return InvalidArgumentErrorBuilder(IREE_LOC) << "Can't parse LLVMIR Module";
  auto dataLayout = module->getDataLayout();
  const auto entry_points = module_def->entry_points();
  llvm::orc::ThreadSafeModule thread_safe_module(std::move(module),
                                                 std::move(llvm_context));
  llvm::Error err =
      execution_engine->addIRModule(std::move(thread_safe_module));
  if (err)
    return InvalidArgumentErrorBuilder(IREE_LOC)
           << "Can't add executable module to execution engine:"
           << llvm::toString(std::move(err));

  auto dylib_serarch_generator =
      llvm::orc::DynamicLibrarySearchGenerator::GetForCurrentProcess(
          dataLayout.getGlobalPrefix());
  if (!dylib_serarch_generator)
    return UnavailableErrorBuilder(IREE_LOC)
           << "Can't resolve symbols in current process";

  auto& main_jitdylib = execution_engine->getMainJITDylib();
  main_jitdylib.addGenerator(std::move(dylib_serarch_generator.get()));

  auto executable =
      make_ref<LLVMJITExecutable>(allocator, spec, allow_aliasing_data);

  for (const auto func_name : *entry_points) {
    auto func_symbol = execution_engine->lookup("invoke_" + func_name->str());
    if (!func_symbol) {
      return NotFoundErrorBuilder(IREE_LOC)
             << "Can't JIT compile function : " << func_name;
    }
    // Map function to its invoke_ symbol.
    executable->InsertSymbol(func_symbol.get());
  }

  return executable;
}

void LLVMJITExecutable::InsertSymbol(llvm::JITEvaluatedSymbol symbol) {
  symbols_.push_back(symbol);
}

Status LLVMJITExecutable::Invoke(int func_id,
                                 llvm::MutableArrayRef<void*> args) {
  const auto func_symbol = symbols_[func_id];
  auto exe_func = (void (*)(void**))func_symbol.getAddress();
  exe_func(args.data());
  return OkStatus();
}

LLVMJITExecutable::LLVMJITExecutable(hal::Allocator* allocator,
                                     ExecutableSpec spec,
                                     bool allow_aliasing_data)
    : spec_(spec) {
  if (!allow_aliasing_data) {
    // Clone data.
    cloned_executable_data_ = {spec.executable_data.begin(),
                               spec.executable_data.end()};
    spec_.executable_data = absl::MakeConstSpan(cloned_executable_data_);
  }
}

LLVMJITExecutable::~LLVMJITExecutable() = default;

}  // namespace llvmjit
}  // namespace hal
}  // namespace iree
<|endoftext|>


<commit_before>#include <iostream>
#include <fstream>
#include <iomanip>
#include <vector>
#include <string>

#include "dg/algorithm.h"
#include "dg/backend/interpolation.cuh"
#include "dg/backend/xspacelib.cuh"
#include "dg/functors.h"
#include "file/read_input.h"
#include "file/nc_utilities.h"
// #include <thrust/random/linear_congruential_engine.h>
// #include <thrust/random/normal_distribution.h>

/**
 * @brief returns histogram
 * @tparam container
 */
template <class container = thrust::host_vector<double> >
struct Histogram
{
     /**
     * @brief Construct from number of bins and input vector
     * @param g1d   grid of output vector
     * @param input input vector
     */
    Histogram(const dg::Grid1d<double>& g1d, const std::vector<double>& in) :
    g1d_(g1d),
    in_(in),
    binwidth_(g1d_.h()),
    count_(dg::evaluate(dg::zero,g1d_))
    {
        for (unsigned j=0;j<in_.size();j++)
        {
            unsigned bin =(unsigned) ((in_[j]-1e-14-g1d_.x0())/binwidth_) ;
            count_[bin ]+=1.;
//             #ifdef DGDEBUG
            std::cout << "input[" << j << "] = " << in_[j] <<
                         " bin = " << bin << " bincount = " << count_[bin ]<<std::endl;
//             #endif
        }
        //Normalize
        unsigned Ampmax = (unsigned)thrust::reduce( count_.begin(), count_.end(),0.,   thrust::maximum<double>()  );
        dg::blas1::scal(count_,1./Ampmax);

    }
    double binwidth() {return binwidth_;}
    double operator()(double x)
    {
        double bin = (unsigned) ((x-1e-14-g1d_.x0())/binwidth_+0.5);
        std::cout <<"x ="<<  x +0.5*binwidth_<< " bin=" <<bin << " count=" << count_[bin] <<std::endl;
        return count_[bin];
    }

    private:
    dg::Grid1d<double> g1d_;
    const std::vector<double> in_;
    double binwidth_;
    container  count_;
};
template <class container = thrust::host_vector<double> >
struct Histogram2D
{
     /**
     * @brief Construct from number of bins and input vector
     * @param g1d   grid of output vector
     * @param input input vector
     */
    Histogram2D(const dg::Grid2d<double>& g2d, const std::vector<double>& inx,const std::vector<double>& iny) :
    g2d_(g2d),
    inx_(inx),
    iny_(iny),
    binwidthx_(g2d_.hx()),
    binwidthy_(g2d_.hy()),
    g1dx_(g2d_.x0(),g2d_.x1(), g2d_.n(), g2d_.Nx(),dg::DIR),
    g1dy_(g2d_.y0(),g2d_.y1(), g2d_.n(), g2d_.Ny(),dg::DIR),
    countx_(dg::evaluate(dg::zero,g1dx_)),
    county_(dg::evaluate(dg::zero,g1dy_)),
    count_(dg::evaluate(dg::zero,g2d_))
    {
        for (unsigned j=0;j<inx_.size();j++)
        {
            unsigned binx =(unsigned) ((inx_[j]-1e-14-g2d_.x0())/binwidthx_) ;
            countx_[binx ]+=1.;
        }

         for (unsigned j=0;j<iny_.size();j++)
        {
            unsigned biny =(unsigned) ((iny_[j]-1e-14-g2d_.y0())/binwidthy_) ;
            county_[biny ]+=1.;
        }

        for (unsigned j=0;j<iny_.size();j++)
        {
            unsigned biny =(unsigned) ((iny_[j]-1e-14-g2d_.y0())/binwidthy_) ;
            for (unsigned i=0;i<inx_.size();i++)
            {
                unsigned binx =(unsigned) ((inx_[i]-1e-14-g2d_.x0())/binwidthx_) ;
//                 std::cout << "x = " << x << " y =" << y << " binx =" << binx <<" biny =" << biny<< std::endl;
//                 if (abs(countx_[binx ] - county_[biny ])<2)  count_[biny*g2d_.Nx()+binx ]=countx_[binx ] + county_[biny ];

//                 if (abs(countx_[binx ] - county_[biny ])<10)  count_[biny*g2d_.Nx()+binx ]+=1.;
                 count_[biny*g2d_.Nx()+binx ]+=1.;
            }
        }
        //Normalize
        unsigned Ampmaxx = (unsigned)thrust::reduce( countx_.begin(), countx_.end(),0.,thrust::maximum<double>()  );
        unsigned Ampmaxy = (unsigned)thrust::reduce( county_.begin(), county_.end(),0.,thrust::maximum<double>()  );
        unsigned Ampmax =  (unsigned)thrust::reduce( count_.begin(),   count_.end(),0.,thrust::maximum<double>()  );
        dg::blas1::scal(countx_, 1./Ampmaxx);
        dg::blas1::scal(county_, 1./Ampmaxy);
        dg::blas1::scal(count_,  1./Ampmax);

    }

    double operator()(double x, double y)
    {
        unsigned binx = (unsigned) ((x-1e-14-g2d_.x0())/binwidthx_+0.5) ;
        unsigned biny = (unsigned) ((y-1e-14-g2d_.y0())/binwidthy_+0.5) ;
//         std::cout << "x = " << x << " y =" << y << " binx =" << binx <<" biny =" << biny<< std::endl;

//         return countx_[binxmom]+county_[binymom];
        return count_[biny*g2d_.Nx()+binx ]; ///(county_[biny ]*countx_[binx ]+1);

    }
    private:
    dg::Grid2d<double> g2d_;
    const std::vector<double> inx_,iny_;
    dg::Grid1d<double> g1dx_,g1dy_;
    double binwidthx_,binwidthy_;
    container countx_,county_;
    container count_;
};
double NormalizeToFluc(std::vector<double>& in) {
    double ex= 0.;
    double exx= 0.;
    double ex2= 0.;
    double sigma = 0.;
    for (unsigned j=0;j<in.size();j++)
    {
        ex+=in[j];
        exx+=in[j]*in[j];
    }
    ex/=in.size();
    exx/=in.size();
    ex2=ex*ex;
    sigma=sqrt(exx-ex2);
    for (unsigned j=0;j<in.size();j++)
    {
        in[j] = (in[j]-  ex)/sigma;
    }
    std::cout << "Sigma = " <<sigma << " Meanvalue = " << ex << std::endl;
    return sigma;
}

int main( int argc, char* argv[])
{

    if( argc != 3)
    {
        std::cerr << "Usage: "<<argv[0]<<" [input.nc] [output.nc]\n";
        return -1;
    }
    std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
    //----------------
    const unsigned Nhist = 30;
    const unsigned nhist = 1;
    const unsigned Ninput =1000;
    std::vector<double> input1(Ninput,0.);
    std::vector<double> input2(Ninput,0.);

    thrust::random::minstd_rand generator;
    thrust::random::normal_distribution<double> d1;
    thrust::random::normal_distribution<double> d2;

    for (unsigned i=0;i<input1.size();i++)  {  input1[i] = d1(generator); }
//     for (unsigned i=0;i<input1.size();i++)  {  input1[i] = d1(generator)*cos(100.*M_PI*i/input1.size()); }

//     for (unsigned i=0;i<input2.size();i++)  {  input2[i] =input1[i]; }
    for (unsigned i=0;i<input2.size();i++)  {  input2[i] =(d2(generator)-3.)*0.001; }

//         (3.*(input1[i]-2.))*cos(100.*M_PI*i/input2.size());}// d2(generator); }
    //normalize grid and compute sigma
    double sigma_1 = NormalizeToFluc(input1);
    double sigma_2 = NormalizeToFluc(input2);
    dg::Grid1d<double>  g1d1(-4.,4., nhist, Nhist,dg::DIR);
    dg::Grid1d<double>  g1d2(-4.,4., nhist, Nhist,dg::DIR);
    dg::Grid2d<double>  g2d( -4.,4.,-4.,4., nhist, Nhist,Nhist,dg::DIR,dg::DIR);
    Histogram<dg::HVec> hist1(g1d1,input1);
    Histogram<dg::HVec> hist2(g1d2,input2);
    Histogram2D<dg::HVec> hist12(g2d,input1,input2);

    dg::HVec PA1 = dg::evaluate(hist1,g1d1);
    dg::HVec A1 = dg::evaluate(dg::coo1,g1d1);
    dg::HVec PA2= dg::evaluate(hist2,g1d2);
    dg::HVec A2 = dg::evaluate(dg::coo1,g1d2);
    dg::HVec PA1A2= dg::evaluate(hist12,g2d);

    //-----------------NC output start
    int dataIDs1[2],dataIDs2[2],dataIDs12[1];
    int dim_ids1[1],dim_ids2[1],dim_ids12[2];
    int ncid;
    file::NC_Error_Handle err;
    err = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);
    //plot 1
    err = file::define_dimension( ncid,"A1_", &dim_ids1[0],  g1d1);
    err = nc_def_var( ncid, "P(A1)",   NC_DOUBLE, 1, &dim_ids1[0], &dataIDs1[0]);
    err = nc_def_var( ncid, "A1",    NC_DOUBLE, 1, &dim_ids1[0], &dataIDs1[1]);
    err = nc_enddef( ncid);
    err = nc_put_var_double( ncid, dataIDs1[0], PA1.data() );
    err = nc_put_var_double( ncid, dataIDs1[1], A1.data() );
    err = nc_redef(ncid);
    //plot 2
    err = file::define_dimension( ncid,"A2_", &dim_ids2[0],  g1d2);
    err = nc_def_var( ncid, "P(A2)",   NC_DOUBLE, 1, &dim_ids2[0], &dataIDs2[0]);
    err = nc_def_var( ncid, "A2",    NC_DOUBLE, 1, &dim_ids2[0], &dataIDs2[1]);
    err = nc_enddef( ncid);
    err = nc_put_var_double( ncid, dataIDs2[0], PA2.data() );
    err = nc_put_var_double( ncid, dataIDs2[1], A2.data() );
    err = nc_redef(ncid);
    //plot12
//     dim_ids12[0]=dim_ids1[0];
//     dim_ids12[1]=dim_ids2[0];
    dim_ids12[0]=dataIDs1[0];
    dim_ids12[1]=dataIDs2[0];
    err = file::define_dimensions( ncid, &dim_ids12[0],  g2d);
    err = nc_def_var( ncid, "P(A1,A2)",   NC_DOUBLE, 2, &dim_ids12[0], &dataIDs12[0]);
    err = nc_enddef( ncid);
    err = nc_put_var_double( ncid, dataIDs12[0], PA1A2.data() );
    err = nc_redef(ncid);
  /*  err = file::define_dimensions( ncid, &dim_ids2[0],  g1d2);
    err = nc_def_var( ncid, names[1].data(), NC_DOUBLE, 1,&dim_ids2[0], &dataIDs[1]);
    err = nc_put_vara_double( ncid, dataIDs[1], hist2g1d2.data());    */
            nc_close( ncid);

//     size_t count[2] = {1, Nhist};
//     size_t start[2] = {0, 0};
   //-----------------NC end
//     dg::HVec hist12g2d = dg::evaluate(hist12,g2d);
//     for (unsigned j=0; j < g1d1.size();j++)
//     {
//         std::cout << "PA1[" << j << "] = " << PA1[j] << std::endl;
//     }
//     for (unsigned j=0; j < g1d2.size();j++)
//     {
//         std::cout << "PA2[" << j << "] = " << PA2[j] << std::endl;
//     }
//     for (unsigned j=0; j < g1d1.size();j++)
//     {
//         for (unsigned i=0; i < g1d2.size();i++)
//         {
//         std::cout << "hist12g1d2[" << j << i << "] = " << hist12g2d[j+i*Nhist] << std::endl;
//         }
//     }
    return 0;
}

<commit_msg>histdiag now correct<commit_after>#include <iostream>
#include <fstream>
#include <iomanip>
#include <vector>
#include <string>
#include <algorithm>

#include "dg/algorithm.h"
#include "dg/backend/interpolation.cuh"
#include "dg/backend/xspacelib.cuh"
#include "dg/functors.h"
#include "file/read_input.h"
#include "file/nc_utilities.h"

/**
 * @brief returns histogram
 * @tparam container
 */
template <class container = thrust::host_vector<double> >
struct Histogram
{
     /**
     * @brief Construct from number of bins and input vector
     * @param g1d   grid of output vector
     * @param in input vector
     */
    Histogram(const dg::Grid1d<double>& g1d, const std::vector<double>& in) :
    g1d_(g1d),
    in_(in),
    binwidth_(g1d_.h()),
    count_(dg::evaluate(dg::zero,g1d_))
    {
        for (unsigned j=0;j<in_.size();j++)
        {
            unsigned bin =floor( (in_[j]-g1d_.x0())/binwidth_ );
            bin = std::max(bin,(unsigned) 0);
            bin = std::min(bin,(unsigned)(g1d_.size()-1));
            count_[bin ]+=1.;
        }
        //Normalize
        unsigned Ampmax = (unsigned)thrust::reduce( count_.begin(), count_.end(),0.,   thrust::maximum<double>()  );
        dg::blas1::scal(count_,1./Ampmax);

    }
    double binwidth() {return binwidth_;}
    double operator()(double x)
    {
        unsigned bin = floor((x-g1d_.x0())/binwidth_+0.5);
        bin = std::max(bin,(unsigned) 0);
        bin = std::min(bin,(unsigned)(g1d_.size()-1));
        return count_[bin];
    }

    private:
    dg::Grid1d<double> g1d_;
    const std::vector<double> in_;
    double binwidth_;
    container  count_;
};
template <class container = thrust::host_vector<double> >
struct Histogram2D
{
     /**
     * @brief Construct from number of bins and input vector
     * @param g2d   grid of output vector
     * @param inx input vector in x direction
     * @param iny input vector in y direction
     */
    Histogram2D(const dg::Grid2d<double>& g2d, const std::vector<double>& inx,const std::vector<double>& iny) :
    g2d_(g2d),
    inx_(inx),
    iny_(iny),
    binwidthx_(g2d_.hx()),
    binwidthy_(g2d_.hy()),
    count_(dg::evaluate(dg::zero,g2d_))
    {

        for (unsigned j=0;j<iny_.size();j++)
        {
            unsigned biny =floor((iny_[j]-g2d_.y0())/binwidthy_) ;
            biny = std::max(biny,(unsigned) 0);
            biny = std::min(biny,(unsigned)(g2d_.Ny()-1));

                unsigned binx =floor((inx_[j]-g2d_.x0())/binwidthx_) ;
                binx = std::max(binx,(unsigned) 0);
                binx = std::min(binx,(unsigned)(g2d_.Nx()-1));
                count_[biny*g2d_.Nx()+binx ]+=1.;

        }
        //Normalize
        unsigned Ampmax =  (unsigned)thrust::reduce( count_.begin(),   count_.end(),0.,thrust::maximum<double>()  );
        dg::blas1::scal(count_,  1./Ampmax);

    }

    double operator()(double x, double y)
    {
        unsigned binx = floor((x-g2d_.x0())/binwidthx_+0.5) ;
        binx = std::max(binx,(unsigned) 0);
        binx = std::min(binx,(unsigned)(g2d_.Nx()-1));
        unsigned biny = floor((y-g2d_.y0())/binwidthy_+0.5) ;
        biny = std::max(biny,(unsigned) 0);
        biny = std::min(biny,(unsigned)(g2d_.Ny()-1));
        return count_[biny*g2d_.Nx()+binx ];

    }
    private:
    dg::Grid2d<double> g2d_;
    const std::vector<double> inx_,iny_;
    double binwidthx_,binwidthy_;
    container count_;
};
/**
 * @brief normalizes input vector
 */
void NormalizeToFluc(std::vector<double>& in) {
    double ex= 0.;
    double exx= 0.;
    double ex2= 0.;
    double sigma = 0.;
    for (unsigned j=0;j<in.size();j++)
    {
        ex+=in[j];
        exx+=in[j]*in[j];
    }
    ex/=in.size();
    exx/=in.size();
    ex2=ex*ex;
    sigma=sqrt(exx-ex2);
    for (unsigned j=0;j<in.size();j++)
    {
        in[j] = (in[j]-  ex)/sigma;
    }
    std::cout << "Sigma = " <<sigma << " Meanvalue = " << ex << std::endl;
}

int main( int argc, char* argv[])
{

    if( argc != 3)
    {
        std::cerr << "Usage: "<<argv[0]<<" [input.nc] [output.nc]\n";
        return -1;
    }
    std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
    //----------------
    const unsigned Nhist = 100;
    const unsigned nhist = 1;
    const unsigned Ninput =50000;
    const double Nsigma =4.;
    std::vector<double> input1(Ninput,0.);
    std::vector<double> input2(Ninput,0.);

    thrust::random::minstd_rand generator;
    thrust::random::normal_distribution<double> d1;
    thrust::random::normal_distribution<double> d2;
    std::vector<double> rand1(Ninput,0.);
    std::vector<double> rand2(Ninput,0.);
    for (unsigned i=0;i<rand1.size();i++)  {  rand1[i] = d1(generator); }
    for (unsigned i=0;i<rand2.size();i++)  {  rand2[i] = d2(generator); }

    for (unsigned i=0;i<input1.size();i++)  {
        double t = (double)(i/(input1.size()-1));
        double omega1 =2.*M_PI* 20.;
        input1[i] = (rand1[i]*0.1*cos( omega1*t)+1.);
    }
    for (unsigned i=0;i<input2.size();i++)  {
        double t = (double)(i/(input2.size()-1));
        double omega1 = 2.*M_PI*20.;
        double omega2= 2.*M_PI*30.;
        double phase = 0.5*M_PI;
//         input2[i] =input1[i];  //perfectly correlated
        input2[i] = (-rand1[i]*0.1*cos(omega1*t)+1.);//perfectly anticorrelated
//         input2[i] = (rand2[i]*0.001*cos(omega2*t)+3.);//perfectly uncorrelated
//         input2[i] = (rand2[i]*0.001*cos(omega2*t)+3.);//uncorrelated
    }

    //normalize grid and compute sigma
    NormalizeToFluc(input1);
    NormalizeToFluc(input2);
    dg::Grid1d<double>  g1d1(-Nsigma,Nsigma, nhist, Nhist,dg::DIR);
    dg::Grid1d<double>  g1d2(-Nsigma,Nsigma, nhist, Nhist,dg::DIR);
    dg::Grid2d<double>  g2d( -Nsigma,Nsigma,-Nsigma,Nsigma, nhist, Nhist,Nhist,dg::DIR,dg::DIR);
    Histogram<dg::HVec> hist1(g1d1,input1);
    Histogram<dg::HVec> hist2(g1d2,input2);
    Histogram2D<dg::HVec> hist12(g2d,input1,input2);

    dg::HVec PA1 = dg::evaluate(hist1,g1d1);
    dg::HVec A1 = dg::evaluate(dg::coo1,g1d1);
    dg::HVec PA2= dg::evaluate(hist2,g1d2);
    dg::HVec A2 = dg::evaluate(dg::coo1,g1d2);
    dg::HVec PA1A2= dg::evaluate(hist12,g2d);

    //-----------------NC output start
    int dataIDs1[2],dataIDs2[2],dataIDs12[1];
    int dim_ids1[1],dim_ids2[1],dim_ids12[2];
    int ncid;
    file::NC_Error_Handle err;
    err = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);
    //plot 1
    err = file::define_dimension( ncid,"A1_", &dim_ids1[0],  g1d1);
    err = nc_def_var( ncid, "P(A1)",   NC_DOUBLE, 1, &dim_ids1[0], &dataIDs1[0]);
    err = nc_def_var( ncid, "A1",    NC_DOUBLE, 1, &dim_ids1[0], &dataIDs1[1]);
    err = nc_enddef( ncid);
    err = nc_put_var_double( ncid, dataIDs1[0], PA1.data() );
    err = nc_put_var_double( ncid, dataIDs1[1], A1.data() );
    err = nc_redef(ncid);
    //plot 2
    err = file::define_dimension( ncid,"A2_", &dim_ids2[0],  g1d2);
    err = nc_def_var( ncid, "P(A2)",   NC_DOUBLE, 1, &dim_ids2[0], &dataIDs2[0]);
    err = nc_def_var( ncid, "A2",    NC_DOUBLE, 1, &dim_ids2[0], &dataIDs2[1]);
    err = nc_enddef( ncid);
    err = nc_put_var_double( ncid, dataIDs2[0], PA2.data() );
    err = nc_put_var_double( ncid, dataIDs2[1], A2.data() );
    err = nc_redef(ncid);
    //plot12
//     dim_ids12[0]=dim_ids1[0];
//     dim_ids12[1]=dim_ids2[0];
    dim_ids12[0]=dataIDs1[0];
    dim_ids12[1]=dataIDs2[0];
    err = file::define_dimensions( ncid, &dim_ids12[0],  g2d);
    err = nc_def_var( ncid, "P(A1,A2)",   NC_DOUBLE, 2, &dim_ids12[0], &dataIDs12[0]);
    err = nc_enddef( ncid);
    err = nc_put_var_double( ncid, dataIDs12[0], PA1A2.data() );
    err = nc_redef(ncid);
    nc_close( ncid);

    return 0;
}

<|endoftext|>


<commit_before>/*
 * ordered_tree.cpp
 *
 *  Created on: 6.01.2015 г.
 *      Author: trifon
 */

#include "bintree.cpp"

template <typename T>
// считаме, че операциите <, >, <=, >=, ==, != дефинират
// линейна наредба над T
class OrderedTree : public BinaryTree<T> {
public:
	T* search(T const& data) const {
		BinaryTreeIterator<T> it = this->iterator();
		while (it && data != *it)
			if (data < *it)
				it = ++it;
			else
				it = it++;
		// когато сме намерили data <-> data == *it
		// ИЛИ
		// когато го няма в дървото <-> !it
		if (!it)
			return NULL;
		return &*it;
	}

	bool addElement(T const& data) {
		TreeNode<T>* p = this->root;
		if (this->root == NULL) {
			this->root = new TreeNode<T>(data);
			return true;
		} else
		while (p->data != data)
			if (data < p->data)
				if (p->left == NULL) {
					p->left = new TreeNode<T>(data);
					return true;
				}
				else
					p = p->left;
			else
				// data > p->data
				if (p->right == NULL) {
					p->right = new TreeNode<T>(data);
					return true;
				}
				else
					p = p->right;
		// когато сме намерили мястото на data и там няма нищо
		// (тогава вмъкваме)
		// p == NULL
		// ИЛИ
		// когато сме намерили data (тогава грешка)
		// p->data == data
		return false;
	}
};

<commit_msg>Реализация на търсене и вмъкване на елемент чрез помощната функция findNode.<commit_after>/*
 * ordered_tree.cpp
 *
 *  Created on: 6.01.2015 г.
 *      Author: trifon
 */

#include "bintree.cpp"

template <typename T>
// считаме, че операциите <, >, <=, >=, ==, != дефинират
// линейна наредба над T
class OrderedTree : public BinaryTree<T> {
private:
	TreeNode<T>*& findNode(T const& data) {
		TreeNode<T>** p = &(this->root);
		while (*p != NULL && (*p)->data != data) {
			if (data < (*p)->data)
				p = &((*p)->left);
			else
				p = &((*p)->right);
		}
		return *p;
	}

public:
	/*
	T* search(T const& data) const {
		BinaryTreeIterator<T> it = this->iterator();
		while (it && data != *it)
			if (data < *it)
				it = ++it;
			else
				it = it++;
		// когато сме намерили data <-> data == *it
		// ИЛИ
		// когато го няма в дървото <-> !it
		if (!it)
			return NULL;
		return &*it;
	}
	*/

	T* search(T const& data) {
		TreeNode<T>*& p = findNode(data);
		if (p == NULL)
			return NULL;
		return &p->data;
	}

	/*
	bool addElement(T const& data) {
		TreeNode<T>* p = this->root;
		if (this->root == NULL) {
			this->root = new TreeNode<T>(data);
			return true;
		} else
		while (p->data != data)
			if (data < p->data)
				if (p->left == NULL) {
					p->left = new TreeNode<T>(data);
					return true;
				}
				else
					p = p->left;
			else
				// data > p->data
				if (p->right == NULL) {
					p->right = new TreeNode<T>(data);
					return true;
				}
				else
					p = p->right;
		// когато сме намерили мястото на data и там няма нищо
		// (тогава вмъкваме)
		// p == NULL
		// ИЛИ
		// когато сме намерили data (тогава грешка)
		// p->data == data
		return false;
	}
	*/

	bool addElement(T const& data) {
		TreeNode<T>*& p = findNode(data);
		if (p != NULL)
			return false;
		p = new TreeNode<T>(data);
		return true;
	}
};

<|endoftext|>


<commit_before>/*
 * Copyright © 2012 Intel Corporation
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library. If not, see <http://www.gnu.org/licenses/>.
 */

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <iostream>
#include <sstream>
#include <set>
#include "llvm/IR/Function.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Module.h"
#include "llvm/IRReader/IRReader.h"
#include "llvm/PassManager.h"
#include "llvm/Pass.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Bitcode/ReaderWriter.h"
#include "llvm/Transforms/IPO.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Support/SourceMgr.h"

#include "sys/cvar.hpp"
#include "src/GBEConfig.h"
#include "llvm/llvm_gen_backend.hpp"
#include "llvm-c/Linker.h"

using namespace llvm;

SVAR(OCL_BITCODE_LIB_PATH, OCL_BITCODE_BIN);

namespace gbe
{
  static Module* createOclBitCodeModule(LLVMContext& ctx, bool strictMath)
  {
    std::string bitCodeFiles = OCL_BITCODE_LIB_PATH;
    std::istringstream bitCodeFilePath(bitCodeFiles);
    std::string FilePath;
    bool findBC = false;
    Module* oclLib = NULL;
    SMDiagnostic Err;

    while (std::getline(bitCodeFilePath, FilePath, ':')) {
      if(access(FilePath.c_str(), R_OK) == 0) {
        findBC = true;
        break;
      }
    }
    assert(findBC);

#if LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR <= 5
    oclLib = getLazyIRFileModule(FilePath, Err, ctx);
#else
    oclLib = getLazyIRFileModule(FilePath, Err, ctx).release();
#endif
    if (!oclLib) {
      printf("Fatal Error: ocl lib can not be opened\n");
      return NULL;
    }

    llvm::GlobalVariable* mathFastFlag = oclLib->getGlobalVariable("__ocl_math_fastpath_flag");
    assert(mathFastFlag);
    Type* intTy = IntegerType::get(ctx, 32);
    mathFastFlag->setInitializer(ConstantInt::get(intTy, strictMath ? 0 : 1));

    return oclLib;
  }

  static bool materializedFuncCall(Module& src, Module& lib, llvm::Function &KF, std::set<std::string>& MFS)
  {
    bool fromSrc = false;
    for (llvm::Function::iterator B = KF.begin(), BE = KF.end(); B != BE; B++) {
      for (BasicBlock::iterator instI = B->begin(),
           instE = B->end(); instI != instE; ++instI) {
        llvm::CallInst* call = dyn_cast<llvm::CallInst>(instI);
        if (!call) {
          continue;
        }

        if (call->getCalledFunction() &&
            call->getCalledFunction()->getIntrinsicID() != 0)
          continue;

        Value *Callee = call->getCalledValue();
        const std::string fnName = Callee->getName();

        if (!MFS.insert(fnName).second) {
          continue;
        }

        fromSrc = false;
        llvm::Function *newMF = lib.getFunction(fnName);
        if (!newMF) {
          newMF = src.getFunction(fnName);
          if (!newMF) {
            printf("Can not find the lib: %s\n", fnName.c_str());
            return false;
          }
          fromSrc = true;
        }

        std::string ErrInfo;// = "Not Materializable";
        if (!fromSrc && newMF->isMaterializable()) {
#if LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR <= 5
          if (newMF->Materialize(&ErrInfo)) {
            printf("Can not materialize the function: %s, because %s\n", fnName.c_str(), ErrInfo.c_str());
            return false;
          }
#else
          if (std::error_code EC = newMF->materialize()) {
            printf("Can not materialize the function: %s, because %s\n", fnName.c_str(), EC.message().c_str());
            return false;
          }
#endif
        }
        if (!materializedFuncCall(src, lib, *newMF, MFS))
          return false;

      }
    }

    return true;
  }

  Module* runBitCodeLinker(Module *mod, bool strictMath)
  {
    LLVMContext& ctx = mod->getContext();
    std::set<std::string> materializedFuncs;
    Module* clonedLib = createOclBitCodeModule(ctx, strictMath);
    assert(clonedLib && "Can not create the beignet bitcode\n");

    std::vector<const char *> kernels;
    std::vector<const char *> builtinFuncs;
    /* Add the memset and memcpy functions here. */
    builtinFuncs.push_back("__gen_memcpy_gg");
    builtinFuncs.push_back("__gen_memcpy_gp");
    builtinFuncs.push_back("__gen_memcpy_gl");
    builtinFuncs.push_back("__gen_memcpy_pg");
    builtinFuncs.push_back("__gen_memcpy_pp");
    builtinFuncs.push_back("__gen_memcpy_pl");
    builtinFuncs.push_back("__gen_memcpy_lg");
    builtinFuncs.push_back("__gen_memcpy_lp");
    builtinFuncs.push_back("__gen_memcpy_ll");
    builtinFuncs.push_back("__gen_memset_p");
    builtinFuncs.push_back("__gen_memset_g");
    builtinFuncs.push_back("__gen_memset_l");

    builtinFuncs.push_back("__gen_memcpy_gg_align");
    builtinFuncs.push_back("__gen_memcpy_gp_align");
    builtinFuncs.push_back("__gen_memcpy_gl_align");
    builtinFuncs.push_back("__gen_memcpy_pg_align");
    builtinFuncs.push_back("__gen_memcpy_pp_align");
    builtinFuncs.push_back("__gen_memcpy_pl_align");
    builtinFuncs.push_back("__gen_memcpy_lg_align");
    builtinFuncs.push_back("__gen_memcpy_lp_align");
    builtinFuncs.push_back("__gen_memcpy_ll_align");
    builtinFuncs.push_back("__gen_memset_p_align");
    builtinFuncs.push_back("__gen_memset_g_align");
    builtinFuncs.push_back("__gen_memset_l_align");

    builtinFuncs.push_back("__gen_memcpy_pc");
    builtinFuncs.push_back("__gen_memcpy_gc");
    builtinFuncs.push_back("__gen_memcpy_lc");

    builtinFuncs.push_back("__gen_memcpy_pc_align");
    builtinFuncs.push_back("__gen_memcpy_gc_align");
    builtinFuncs.push_back("__gen_memcpy_lc_align");

    for (Module::iterator SF = mod->begin(), E = mod->end(); SF != E; ++SF) {
      if (SF->isDeclaration()) continue;
      if (!isKernelFunction(*SF)) continue;
      kernels.push_back(SF->getName().data());

      if (!materializedFuncCall(*mod, *clonedLib, *SF, materializedFuncs)) {
        delete clonedLib;
        return NULL;
      }
    }

    if (kernels.empty()) {
      printf("One module without kernel function!\n");
      delete clonedLib;
      return NULL;
    }

    for (auto &f : builtinFuncs) {
      const std::string fnName(f);
      if (!materializedFuncs.insert(fnName).second) {
        continue;
      }

      llvm::Function *newMF = clonedLib->getFunction(fnName);
      if (!newMF) {
        printf("Can not find the function: %s\n", fnName.c_str());
        delete clonedLib;
        return NULL;
      }
      std::string ErrInfo;// = "Not Materializable";
      if (newMF->isMaterializable()) {
#if LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR <= 5
        if (newMF->Materialize(&ErrInfo)) {
          printf("Can not materialize the function: %s, because %s\n", fnName.c_str(), ErrInfo.c_str());
          delete clonedLib;
          return NULL;
        }
      }
#else
        if (std::error_code EC = newMF->materialize()) {
          printf("Can not materialize the function: %s, because %s\n", fnName.c_str(), EC.message().c_str());
          delete clonedLib;
          return NULL;
        }
      }
#endif

      if (!materializedFuncCall(*mod, *clonedLib, *newMF, materializedFuncs)) {
        delete clonedLib;
        return NULL;
      }

      kernels.push_back(f);
    }

    /* We use beignet's bitcode as dst because it will have a lot of
       lazy functions which will not be loaded. */
    char* errorMsg;
    if(LLVMLinkModules(wrap(clonedLib), wrap(mod), LLVMLinkerDestroySource, &errorMsg)) {
      delete clonedLib;
      printf("Fatal Error: link the bitcode error:\n%s\n", errorMsg);
      return NULL;
    }

    llvm::PassManager passes;

    passes.add(createInternalizePass(kernels));
    passes.add(createGlobalDCEPass());

    passes.run(*clonedLib);

    return clonedLib;
  }

} // end namespace
<commit_msg>reset the SPIR target datalayout.<commit_after>/*
 * Copyright © 2012 Intel Corporation
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library. If not, see <http://www.gnu.org/licenses/>.
 */

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <iostream>
#include <sstream>
#include <set>
#include "llvm/IR/Function.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Module.h"
#include "llvm/IRReader/IRReader.h"
#include "llvm/PassManager.h"
#include "llvm/Pass.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Bitcode/ReaderWriter.h"
#include "llvm/Transforms/IPO.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Support/SourceMgr.h"

#include "sys/cvar.hpp"
#include "src/GBEConfig.h"
#include "llvm/llvm_gen_backend.hpp"
#include "llvm-c/Linker.h"

using namespace llvm;

SVAR(OCL_BITCODE_LIB_PATH, OCL_BITCODE_BIN);

namespace gbe
{
  static Module* createOclBitCodeModule(LLVMContext& ctx, bool strictMath)
  {
    std::string bitCodeFiles = OCL_BITCODE_LIB_PATH;
    std::istringstream bitCodeFilePath(bitCodeFiles);
    std::string FilePath;
    bool findBC = false;
    Module* oclLib = NULL;
    SMDiagnostic Err;

    while (std::getline(bitCodeFilePath, FilePath, ':')) {
      if(access(FilePath.c_str(), R_OK) == 0) {
        findBC = true;
        break;
      }
    }
    assert(findBC);

#if LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR <= 5
    oclLib = getLazyIRFileModule(FilePath, Err, ctx);
#else
    oclLib = getLazyIRFileModule(FilePath, Err, ctx).release();
#endif
    if (!oclLib) {
      printf("Fatal Error: ocl lib can not be opened\n");
      return NULL;
    }

    llvm::GlobalVariable* mathFastFlag = oclLib->getGlobalVariable("__ocl_math_fastpath_flag");
    assert(mathFastFlag);
    Type* intTy = IntegerType::get(ctx, 32);
    mathFastFlag->setInitializer(ConstantInt::get(intTy, strictMath ? 0 : 1));

    return oclLib;
  }

  static bool materializedFuncCall(Module& src, Module& lib, llvm::Function &KF, std::set<std::string>& MFS)
  {
    bool fromSrc = false;
    for (llvm::Function::iterator B = KF.begin(), BE = KF.end(); B != BE; B++) {
      for (BasicBlock::iterator instI = B->begin(),
           instE = B->end(); instI != instE; ++instI) {
        llvm::CallInst* call = dyn_cast<llvm::CallInst>(instI);
        if (!call) {
          continue;
        }

        if (call->getCalledFunction() &&
            call->getCalledFunction()->getIntrinsicID() != 0)
          continue;

        Value *Callee = call->getCalledValue();
        const std::string fnName = Callee->getName();

        if (!MFS.insert(fnName).second) {
          continue;
        }

        fromSrc = false;
        llvm::Function *newMF = lib.getFunction(fnName);
        if (!newMF) {
          newMF = src.getFunction(fnName);
          if (!newMF) {
            printf("Can not find the lib: %s\n", fnName.c_str());
            return false;
          }
          fromSrc = true;
        }

        std::string ErrInfo;// = "Not Materializable";
        if (!fromSrc && newMF->isMaterializable()) {
#if LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR <= 5
          if (newMF->Materialize(&ErrInfo)) {
            printf("Can not materialize the function: %s, because %s\n", fnName.c_str(), ErrInfo.c_str());
            return false;
          }
#else
          if (std::error_code EC = newMF->materialize()) {
            printf("Can not materialize the function: %s, because %s\n", fnName.c_str(), EC.message().c_str());
            return false;
          }
#endif
        }
        if (!materializedFuncCall(src, lib, *newMF, MFS))
          return false;

      }
    }

    return true;
  }

  Module* runBitCodeLinker(Module *mod, bool strictMath)
  {
    LLVMContext& ctx = mod->getContext();
    std::set<std::string> materializedFuncs;
    Module* clonedLib = createOclBitCodeModule(ctx, strictMath);
    assert(clonedLib && "Can not create the beignet bitcode\n");

    std::vector<const char *> kernels;
    std::vector<const char *> builtinFuncs;
    /* Add the memset and memcpy functions here. */
    builtinFuncs.push_back("__gen_memcpy_gg");
    builtinFuncs.push_back("__gen_memcpy_gp");
    builtinFuncs.push_back("__gen_memcpy_gl");
    builtinFuncs.push_back("__gen_memcpy_pg");
    builtinFuncs.push_back("__gen_memcpy_pp");
    builtinFuncs.push_back("__gen_memcpy_pl");
    builtinFuncs.push_back("__gen_memcpy_lg");
    builtinFuncs.push_back("__gen_memcpy_lp");
    builtinFuncs.push_back("__gen_memcpy_ll");
    builtinFuncs.push_back("__gen_memset_p");
    builtinFuncs.push_back("__gen_memset_g");
    builtinFuncs.push_back("__gen_memset_l");

    builtinFuncs.push_back("__gen_memcpy_gg_align");
    builtinFuncs.push_back("__gen_memcpy_gp_align");
    builtinFuncs.push_back("__gen_memcpy_gl_align");
    builtinFuncs.push_back("__gen_memcpy_pg_align");
    builtinFuncs.push_back("__gen_memcpy_pp_align");
    builtinFuncs.push_back("__gen_memcpy_pl_align");
    builtinFuncs.push_back("__gen_memcpy_lg_align");
    builtinFuncs.push_back("__gen_memcpy_lp_align");
    builtinFuncs.push_back("__gen_memcpy_ll_align");
    builtinFuncs.push_back("__gen_memset_p_align");
    builtinFuncs.push_back("__gen_memset_g_align");
    builtinFuncs.push_back("__gen_memset_l_align");

    builtinFuncs.push_back("__gen_memcpy_pc");
    builtinFuncs.push_back("__gen_memcpy_gc");
    builtinFuncs.push_back("__gen_memcpy_lc");

    builtinFuncs.push_back("__gen_memcpy_pc_align");
    builtinFuncs.push_back("__gen_memcpy_gc_align");
    builtinFuncs.push_back("__gen_memcpy_lc_align");

    for (Module::iterator SF = mod->begin(), E = mod->end(); SF != E; ++SF) {
      if (SF->isDeclaration()) continue;
      if (!isKernelFunction(*SF)) continue;
      kernels.push_back(SF->getName().data());

      if (!materializedFuncCall(*mod, *clonedLib, *SF, materializedFuncs)) {
        delete clonedLib;
        return NULL;
      }
    }

    if (kernels.empty()) {
      printf("One module without kernel function!\n");
      delete clonedLib;
      return NULL;
    }

    for (auto &f : builtinFuncs) {
      const std::string fnName(f);
      if (!materializedFuncs.insert(fnName).second) {
        continue;
      }

      llvm::Function *newMF = clonedLib->getFunction(fnName);
      if (!newMF) {
        printf("Can not find the function: %s\n", fnName.c_str());
        delete clonedLib;
        return NULL;
      }
      std::string ErrInfo;// = "Not Materializable";
      if (newMF->isMaterializable()) {
#if LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR <= 5
        if (newMF->Materialize(&ErrInfo)) {
          printf("Can not materialize the function: %s, because %s\n", fnName.c_str(), ErrInfo.c_str());
          delete clonedLib;
          return NULL;
        }
      }
#else
        if (std::error_code EC = newMF->materialize()) {
          printf("Can not materialize the function: %s, because %s\n", fnName.c_str(), EC.message().c_str());
          delete clonedLib;
          return NULL;
        }
      }
#endif

      if (!materializedFuncCall(*mod, *clonedLib, *newMF, materializedFuncs)) {
        delete clonedLib;
        return NULL;
      }

      kernels.push_back(f);
    }

    /* the SPIR binary datalayout maybe different with beignet's bitcode */
    if(clonedLib->getDataLayout() != mod->getDataLayout())
      mod->setDataLayout(clonedLib->getDataLayout());

    /* We use beignet's bitcode as dst because it will have a lot of
       lazy functions which will not be loaded. */
    char* errorMsg;
    if(LLVMLinkModules(wrap(clonedLib), wrap(mod), LLVMLinkerDestroySource, &errorMsg)) {
      delete clonedLib;
      printf("Fatal Error: link the bitcode error:\n%s\n", errorMsg);
      return NULL;
    }

    llvm::PassManager passes;

    passes.add(createInternalizePass(kernels));
    passes.add(createGlobalDCEPass());

    passes.run(*clonedLib);

    return clonedLib;
  }

} // end namespace
<|endoftext|>


<commit_before><commit_msg>[vm/profiler] On Android use alternative stack for handling SIGPROF.<commit_after><|endoftext|>