<commit_before>// RUN: %clangxx_msan -O0 -g -DPOSITIVE %s -o %t
// RUN: not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK
// RUN: MSAN_OPTIONS=verbosity=1 not %run %t 2>&1 | \
// RUN:     FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-VERBOSE

// RUN: %clangxx_msan -O0 -g %s -o %t && %run %t

#include <sanitizer/msan_interface.h>

int main(void) {
  char p[32] = {};
  __msan_poison(p + 10, 2);

  __msan_check_mem_is_initialized(p, 10);
  __msan_check_mem_is_initialized(p + 12, 30);
#ifdef POSITIVE
  __msan_check_mem_is_initialized(p + 5, 20);
  // CHECK: Uninitialized bytes in __msan_check_mem_is_initialized at offset 5 inside [0x{{.*}}, 20)
  // CHECK-VERBOSE: Shadow map of [0x{{.*}}, 0x{{.*}}), 20 bytes:
  // CHECK-VERBOSE: 0x{{.*}}: ..000000 0000ffff 00000000 00000000
  // CHECK-VERBOSE: 0x{{.*}}: 00000000 00...... ........ ........

  // CHECK: WARNING: MemorySanitizer: use-of-uninitialized-value
  // CHECK: #0 0x{{.*}}in main{{.*}}msan_check_mem_is_initialized.cc:[[@LINE-7]]
#endif
  return 0;
}

<commit_msg>[compiler-rt] Remove redundant --check-prefix=CHECK from test<commit_after>// RUN: %clangxx_msan -O0 -g -DPOSITIVE %s -o %t
// RUN: not %run %t 2>&1 | FileCheck %s
// RUN: MSAN_OPTIONS=verbosity=1 not %run %t 2>&1 | \
// RUN:     FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-VERBOSE

// RUN: %clangxx_msan -O0 -g %s -o %t && %run %t

#include <sanitizer/msan_interface.h>

int main(void) {
  char p[32] = {};
  __msan_poison(p + 10, 2);

  __msan_check_mem_is_initialized(p, 10);
  __msan_check_mem_is_initialized(p + 12, 30);
#ifdef POSITIVE
  __msan_check_mem_is_initialized(p + 5, 20);
  // CHECK: Uninitialized bytes in __msan_check_mem_is_initialized at offset 5 inside [0x{{.*}}, 20)
  // CHECK-VERBOSE: Shadow map of [0x{{.*}}, 0x{{.*}}), 20 bytes:
  // CHECK-VERBOSE: 0x{{.*}}: ..000000 0000ffff 00000000 00000000
  // CHECK-VERBOSE: 0x{{.*}}: 00000000 00...... ........ ........

  // CHECK: WARNING: MemorySanitizer: use-of-uninitialized-value
  // CHECK: #0 0x{{.*}}in main{{.*}}msan_check_mem_is_initialized.cc:[[@LINE-7]]
#endif
  return 0;
}

<|endoftext|>


<commit_before>/*
 * Copyright (C) 2012 by Glenn Hickey (hickey@soe.ucsc.edu)
 *
 * Released under the MIT license, see LICENSE.txt
 */

#include <cstdlib>
#include <iostream>
#include <fstream>
#include "halMafExport.h"

using namespace std;
using namespace hal;

static CLParserPtr initParser()
{
  CLParserPtr optionsParser = hdf5CLParserInstance();
  optionsParser->addArgument("halFile", "input hal file");
  optionsParser->addArgument("mafFile", "output maf file");
  optionsParser->addOption("refGenome",
                           "name of reference genome (root if empty)",
                           "\"\"");
  optionsParser->addOption("refSequence",
                           "name of reference sequence within reference genome"
                           " (all sequences if empty)",
                           "\"\"");
  optionsParser->addOption("start",
                           "coordinate within reference genome (or sequence"
                           " if specified) to start at",
                           0);
  optionsParser->addOption("length",
                           "length of the reference genome (or sequence"
                           " if specified) to convert.  If set to 0,"
                           " the entire thing is converted",
                           0);
  optionsParser->addOption("rootGenome",
                           "name of root genome (none if empty)",
                           "\"\"");
  optionsParser->addOption("targetGenomes",
                           "comma-separated (no spaces) list of target genomes "
                           "(others are excluded) (vist all if empty)",
                           "\"\"");
  optionsParser->addOption("maxRefGap",
                           "maximum gap length in reference",
                           0);
  optionsParser->addOptionFlag("noDupes",
                               "ignore paralogy edges",
                               false);
  optionsParser->addOptionFlag("noAncestors",
                               "don't write ancestral sequences",
                               false);

  optionsParser->setDescription("Convert hal database to maf.");
  return optionsParser;
}

int main(int argc, char** argv)
{
  CLParserPtr optionsParser = initParser();
  string halPath;
  string mafPath;
  string refGenomeName;
  string rootGenomeName;
  string targetGenomes;
  string refSequenceName;
  hal_index_t start;
  hal_size_t length;
  hal_size_t maxRefGap;
  bool noDupes;
  bool noAncestors;
  try
  {
    optionsParser->parseOptions(argc, argv);
    halPath = optionsParser->getArgument<string>("halFile");
    mafPath = optionsParser->getArgument<string>("mafFile");
    refGenomeName = optionsParser->getOption<string>("refGenome");
    rootGenomeName = optionsParser->getOption<string>("rootGenome");
    targetGenomes = optionsParser->getOption<string>("targetGenomes");
    refSequenceName = optionsParser->getOption<string>("refSequence");
    start = optionsParser->getOption<hal_index_t>("start");
    length = optionsParser->getOption<hal_size_t>("length");
    maxRefGap = optionsParser->getOption<hal_size_t>("maxRefGap");
    noDupes = optionsParser->getFlag("noDupes");
    noAncestors = optionsParser->getFlag("noAncestors");

    if (rootGenomeName != "\"\"" && targetGenomes != "\"\"")
    {
      throw hal_exception("--rootGenome and --targetGenomes options are "
                          " mutually exclusive");
    }
  }
  catch(exception& e)
  {
    cerr << e.what() << endl;
    optionsParser->printUsage(cerr);
    exit(1);
  }
  try
  {
    AlignmentConstPtr alignment = openHalAlignmentReadOnly(halPath,
                                                           optionsParser);
    if (alignment->getNumGenomes() == 0)
    {
      throw hal_exception("hal alignmenet is empty");
    }

    set<const Genome*> targetSet;
    const Genome* rootGenome = NULL;
    if (rootGenomeName != "\"\"")
    {
      rootGenome = alignment->openGenome(rootGenomeName);
      if (rootGenome == NULL)
      {
        throw hal_exception(string("Root genome, ") + rootGenomeName +
                            ", not found in alignment");
      }
      if (rootGenomeName != alignment->getRootName())
      {
        getGenomesInSubTree(rootGenome, targetSet);
      }
    }

    if (targetGenomes != "\"\"")
    {
      vector<string> targetNames = chopString(targetGenomes, ",");
      for (size_t i = 0; i < targetNames.size(); ++i)
      {
        const Genome* tgtGenome = alignment->openGenome(targetNames[i]);
        if (tgtGenome == NULL)
        {
          throw hal_exception(string("Target genome, ") + targetNames[i] +
                              ", not found in alignment");
        }
        targetSet.insert(tgtGenome);
      }
    }

    const Genome* refGenome = NULL;
    if (refGenomeName != "\"\"")
    {
      refGenome = alignment->openGenome(refGenomeName);
      if (refGenome == NULL)
      {
        throw hal_exception(string("Reference genome, ") + refGenomeName +
                            ", not found in alignment");
      }
    }
    else
    {
      refGenome = alignment->openGenome(alignment->getRootName());
    }
    const SegmentedSequence* ref = refGenome;

    const Sequence* refSequence = NULL;
    if (refSequenceName != "\"\"")
    {
      refSequence = refGenome->getSequence(refSequenceName);
      ref = refSequence;
      if (refSequence == NULL)
      {
        throw hal_exception(string("Reference sequence, ") + refSequenceName +
                            ", not found in reference genome, " +
                            refGenome->getName());
      }
    }

    ofstream mafStream(mafPath.c_str());
    if (!mafStream)
    {
      throw hal_exception("Error opening " + mafPath);
    }

    MafExport mafExport;
    mafExport.setMaxRefGap(maxRefGap);
    mafExport.setNoDupes(noDupes);
    mafExport.setNoAncestors(noAncestors);

    mafExport.convertSegmentedSequence(mafStream, alignment, ref,
                                       start, length, targetSet);

  }
  catch(hal_exception& e)
  {
    cerr << "hal exception caught: " << e.what() << endl;
    return 1;
  }
  catch(exception& e)
  {
    cerr << "Exception caught: " << e.what() << endl;
    return 1;
  }

  return 0;
}
<commit_msg>add bed targets option for hal2maf<commit_after>/*
 * Copyright (C) 2012 by Glenn Hickey (hickey@soe.ucsc.edu)
 *
 * Released under the MIT license, see LICENSE.txt
 */

#include <cstdlib>
#include <iostream>
#include <fstream>
#include "halMafExport.h"

using namespace std;
using namespace hal;

static CLParserPtr initParser()
{
  CLParserPtr optionsParser = hdf5CLParserInstance();
  optionsParser->addArgument("halFile", "input hal file");
  optionsParser->addArgument("mafFile", "output maf file");
  optionsParser->addOption("refGenome",
                           "name of reference genome (root if empty)",
                           "\"\"");
  optionsParser->addOption("refSequence",
                           "name of reference sequence within reference genome"
                           " (all sequences if empty)",
                           "\"\"");
  optionsParser->addOption("refTargets",
                           "bed file coordinates of intervals in the reference "
                           "genome to export",
                           "\"\"");
  optionsParser->addOption("start",
                           "coordinate within reference genome (or sequence"
                           " if specified) to start at",
                           0);
  optionsParser->addOption("length",
                           "length of the reference genome (or sequence"
                           " if specified) to convert.  If set to 0,"
                           " the entire thing is converted",
                           0);
  optionsParser->addOption("rootGenome",
                           "name of root genome (none if empty)",
                           "\"\"");
  optionsParser->addOption("targetGenomes",
                           "comma-separated (no spaces) list of target genomes "
                           "(others are excluded) (vist all if empty)",
                           "\"\"");
  optionsParser->addOption("maxRefGap",
                           "maximum gap length in reference",
                           0);
  optionsParser->addOptionFlag("noDupes",
                               "ignore paralogy edges",
                               false);
  optionsParser->addOptionFlag("noAncestors",
                               "don't write ancestral sequences",
                               false);

  optionsParser->setDescription("Convert hal database to maf.");
  return optionsParser;
}

int main(int argc, char** argv)
{
  CLParserPtr optionsParser = initParser();
  string halPath;
  string mafPath;
  string refGenomeName;
  string rootGenomeName;
  string targetGenomes;
  string refSequenceName;
  string refTargetsPath;
  hal_index_t start;
  hal_size_t length;
  hal_size_t maxRefGap;
  bool noDupes;
  bool noAncestors;
  try
  {
    optionsParser->parseOptions(argc, argv);
    halPath = optionsParser->getArgument<string>("halFile");
    mafPath = optionsParser->getArgument<string>("mafFile");
    refGenomeName = optionsParser->getOption<string>("refGenome");
    rootGenomeName = optionsParser->getOption<string>("rootGenome");
    targetGenomes = optionsParser->getOption<string>("targetGenomes");
    refSequenceName = optionsParser->getOption<string>("refSequence");
    refTargetsPath = optionsParser->getOption<string>("refTargets");
    start = optionsParser->getOption<hal_index_t>("start");
    length = optionsParser->getOption<hal_size_t>("length");
    maxRefGap = optionsParser->getOption<hal_size_t>("maxRefGap");
    noDupes = optionsParser->getFlag("noDupes");
    noAncestors = optionsParser->getFlag("noAncestors");

    if (rootGenomeName != "\"\"" && targetGenomes != "\"\"")
    {
      throw hal_exception("--rootGenome and --targetGenomes options are "
                          "mutually exclusive");
    }
    if (refTargetsPath != "\"\"" && refSequenceName != "\"\"")
    {
      throw hal_exception("--refSequence and --refTargets options are "
                          "mutually exclusive");
    }
  }
  catch(exception& e)
  {
    cerr << e.what() << endl;
    optionsParser->printUsage(cerr);
    exit(1);
  }
  try
  {
    AlignmentConstPtr alignment = openHalAlignmentReadOnly(halPath,
                                                           optionsParser);
    if (alignment->getNumGenomes() == 0)
    {
      throw hal_exception("hal alignmenet is empty");
    }

    set<const Genome*> targetSet;
    const Genome* rootGenome = NULL;
    if (rootGenomeName != "\"\"")
    {
      rootGenome = alignment->openGenome(rootGenomeName);
      if (rootGenome == NULL)
      {
        throw hal_exception(string("Root genome, ") + rootGenomeName +
                            ", not found in alignment");
      }
      if (rootGenomeName != alignment->getRootName())
      {
        getGenomesInSubTree(rootGenome, targetSet);
      }
    }

    if (targetGenomes != "\"\"")
    {
      vector<string> targetNames = chopString(targetGenomes, ",");
      for (size_t i = 0; i < targetNames.size(); ++i)
      {
        const Genome* tgtGenome = alignment->openGenome(targetNames[i]);
        if (tgtGenome == NULL)
        {
          throw hal_exception(string("Target genome, ") + targetNames[i] +
                              ", not found in alignment");
        }
        targetSet.insert(tgtGenome);
      }
    }

    const Genome* refGenome = NULL;
    if (refGenomeName != "\"\"")
    {
      refGenome = alignment->openGenome(refGenomeName);
      if (refGenome == NULL)
      {
        throw hal_exception(string("Reference genome, ") + refGenomeName +
                            ", not found in alignment");
      }
    }
    else
    {
      refGenome = alignment->openGenome(alignment->getRootName());
    }
    const SegmentedSequence* ref = refGenome;

    const Sequence* refSequence = NULL;
    if (refSequenceName != "\"\"")
    {
      refSequence = refGenome->getSequence(refSequenceName);
      ref = refSequence;
      if (refSequence == NULL)
      {
        throw hal_exception(string("Reference sequence, ") + refSequenceName +
                            ", not found in reference genome, " +
                            refGenome->getName());
      }
    }

    ofstream mafStream(mafPath.c_str());
    if (!mafStream)
    {
      throw hal_exception("Error opening " + mafPath);
    }

    MafExport mafExport;
    mafExport.setMaxRefGap(maxRefGap);
    mafExport.setNoDupes(noDupes);
    mafExport.setNoAncestors(noAncestors);

    ifstream refTargetsStream;
    if (refTargetsPath != "\"\"")
    {
      refTargetsStream.open(refTargetsPath.c_str());
      if (!refTargetsStream)
      {
        throw hal_exception("Error opening " + refTargetsPath);
      }
      string line;
      hal_size_t end;
      while (!refTargetsStream.bad() && !refTargetsStream.eof())
      {
        getline(refTargetsStream, line);
        istringstream ss(line);
        ss >> refSequenceName >> start >> end;
        if (ss.bad() && (!refSequenceName.empty() || refSequenceName[0] != '#'))
        {
          cerr << "skipping malformed line " << line << " in "
               << refTargetsPath << "\n";
        }
        else
        {
          refSequence = refGenome->getSequence(refSequenceName);
          length = end - start;
          if (refSequence != NULL && length <= refSequence->getSequenceLength())
          {
            start = refSequence->getStartPosition() + start;
            mafExport.convertSegmentedSequence(mafStream, alignment,
                                               refSequence,
                                               start, length, targetSet);
          }
          else
          {
            cerr << "bed coordinate " << refSequenceName << " "
                 << start << " " << end << " not found in "
                 << refGenome->getName() << "\n";
          }
        }
      }
    }
    else
    {
      mafExport.convertSegmentedSequence(mafStream, alignment, ref,
                                         start, length, targetSet);
    }
  }
  catch(hal_exception& e)
  {
    cerr << "hal exception caught: " << e.what() << endl;
    return 1;
  }
  catch(exception& e)
  {
    cerr << "Exception caught: " << e.what() << endl;
    return 1;
  }

  return 0;
}
<|endoftext|>


<commit_before>/*
 * Copyright 2013 Real Logic Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#ifndef _SBE_HPP_
#define _SBE_HPP_

#include <string.h>
#include <stdint.h>
#include <limits.h>
#include <stdexcept>

/*
 * Types used by C++ codec. Might have to be platform specific at some stage.
 */
typedef char sbe_char_t;
typedef ::int8_t sbe_int8_t;
typedef ::int16_t sbe_int16_t;
typedef ::int32_t sbe_int32_t;
typedef ::int64_t sbe_int64_t;
typedef ::uint8_t sbe_uint8_t;
typedef ::uint16_t sbe_uint16_t;
typedef ::uint32_t sbe_uint32_t;
typedef ::uint64_t sbe_uint64_t;
typedef float sbe_float_t;
typedef double sbe_double_t;

namespace sbe {

/*
 * Define some byte ordering macros
 */
#if defined(WIN32) || defined(_WIN32)
    #define SBE_BIG_ENDIAN_ENCODE_16(v) _byteswap_ushort(v)
    #define SBE_BIG_ENDIAN_ENCODE_32(v) _byteswap_ulong(v)
    #define SBE_BIG_ENDIAN_ENCODE_64(v) _byteswap_uint64(v)
    #define SBE_LITTLE_ENDIAN_ENCODE_16(v) (v)
    #define SBE_LITTLE_ENDIAN_ENCODE_32(v) (v)
    #define SBE_LITTLE_ENDIAN_ENCODE_64(v) (v)
#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
    #define SBE_BIG_ENDIAN_ENCODE_16(v) __builtin_bswap16(v)
    #define SBE_BIG_ENDIAN_ENCODE_32(v) __builtin_bswap32(v)
    #define SBE_BIG_ENDIAN_ENCODE_64(v) __builtin_bswap64(v)
    #define SBE_LITTLE_ENDIAN_ENCODE_16(v) (v)
    #define SBE_LITTLE_ENDIAN_ENCODE_32(v) (v)
    #define SBE_LITTLE_ENDIAN_ENCODE_64(v) (v)
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
    #define SBE_LITTLE_ENDIAN_ENCODE_16(v) __builtin_bswap16(v)
    #define SBE_LITTLE_ENDIAN_ENCODE_32(v) __builtin_bswap32(v)
    #define SBE_LITTLE_ENDIAN_ENCODE_64(v) __builtin_bswap64(v)
    #define SBE_BIG_ENDIAN_ENCODE_16(v) (v)
    #define SBE_BIG_ENDIAN_ENCODE_32(v) (v)
    #define SBE_BIG_ENDIAN_ENCODE_64(v) (v)
#else
    #error "Byte Ordering of platform not determined. Set __BYTE_ORDER__ manually before including this file."
#endif

#if defined(SBE_NO_BOUNDS_CHECK)
    #define SBE_BOUNDS_CHECK_EXPECT(exp,c) (false)
#elif defined(_MSC_VER)
    #define SBE_BOUNDS_CHECK_EXPECT(exp,c) (exp)
#else
    #define SBE_BOUNDS_CHECK_EXPECT(exp,c) (__builtin_expect(exp,c))
#endif

#if defined(__GNUC__)
    #define SBE_NULLVALUE_INT8 (INT8_MIN)
    #define SBE_NULLVALUE_INT16 (INT16_MIN)
    #define SBE_NULLVALUE_INT32 (INT32_MIN)
    #define SBE_NULLVALUE_INT64 (INT64_MIN)
    #define SBE_NULLVALUE_UINT8 (UINT8_MAX)
    #define SBE_NULLVALUE_UINT16 (UINT16_MAX)
    #define SBE_NULLVALUE_UINT32 (UINT32_MAX)
    #define SBE_NULLVALUE_UINT64 (UINT64_MAX)
#elif defined(_MSC_VER)
    // Visual C++ does not handle minimum integer values properly
    // See: http://msdn.microsoft.com/en-us/library/4kh09110.aspx
    #define SBE_NULLVALUE_INT8 (SCHAR_MIN)
    #define SBE_NULLVALUE_INT16 (SHRT_MIN)
    #define SBE_NULLVALUE_INT32 (LONG_MIN)
    #define SBE_NULLVALUE_INT64 (LLONG_MIN)
    #define SBE_NULLVALUE_UINT8 (UCHAR_MAX)
    #define SBE_NULLVALUE_UINT16 (USHRT_MAX)
    #define SBE_NULLVALUE_UINT32 (ULONG_MAX)
    #define SBE_NULLVALUE_UINT64 (ULLONG_MAX)
#else
    #define SBE_NULLVALUE_INT8 (INT8_MIN)
    #define SBE_NULLVALUE_INT16 (INT16_MIN)
    #define SBE_NULLVALUE_INT32 (INT32_MIN)
    #define SBE_NULLVALUE_INT64 (INT64_MIN)
    #define SBE_NULLVALUE_UINT8 (UINT8_MAX)
    #define SBE_NULLVALUE_UINT16 (UINT16_MAX)
    #define SBE_NULLVALUE_UINT32 (UINT32_MAX)
    #define SBE_NULLVALUE_UINT64 (UINT64_MAX)
#endif

namespace MetaAttribute {

enum Attribute
{
    EPOCH,
    TIME_UNIT,
    SEMANTIC_TYPE
};

}

}

#endif /* _SBE_HPP_ */
<commit_msg>[C++]: fix #274. Add __STDC_LIMIT_MACROS before inclusion of stdint.h to comply with C99 requirements.<commit_after>/*
 * Copyright 2013 Real Logic Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#ifndef _SBE_HPP_
#define _SBE_HPP_

#include <string.h>
#define __STDC_LIMIT_MACROS
#include <stdint.h>
#include <limits.h>
#include <stdexcept>

/*
 * Types used by C++ codec. Might have to be platform specific at some stage.
 */
typedef char sbe_char_t;
typedef ::int8_t sbe_int8_t;
typedef ::int16_t sbe_int16_t;
typedef ::int32_t sbe_int32_t;
typedef ::int64_t sbe_int64_t;
typedef ::uint8_t sbe_uint8_t;
typedef ::uint16_t sbe_uint16_t;
typedef ::uint32_t sbe_uint32_t;
typedef ::uint64_t sbe_uint64_t;
typedef float sbe_float_t;
typedef double sbe_double_t;

namespace sbe {

/*
 * Define some byte ordering macros
 */
#if defined(WIN32) || defined(_WIN32)
    #define SBE_BIG_ENDIAN_ENCODE_16(v) _byteswap_ushort(v)
    #define SBE_BIG_ENDIAN_ENCODE_32(v) _byteswap_ulong(v)
    #define SBE_BIG_ENDIAN_ENCODE_64(v) _byteswap_uint64(v)
    #define SBE_LITTLE_ENDIAN_ENCODE_16(v) (v)
    #define SBE_LITTLE_ENDIAN_ENCODE_32(v) (v)
    #define SBE_LITTLE_ENDIAN_ENCODE_64(v) (v)
#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
    #define SBE_BIG_ENDIAN_ENCODE_16(v) __builtin_bswap16(v)
    #define SBE_BIG_ENDIAN_ENCODE_32(v) __builtin_bswap32(v)
    #define SBE_BIG_ENDIAN_ENCODE_64(v) __builtin_bswap64(v)
    #define SBE_LITTLE_ENDIAN_ENCODE_16(v) (v)
    #define SBE_LITTLE_ENDIAN_ENCODE_32(v) (v)
    #define SBE_LITTLE_ENDIAN_ENCODE_64(v) (v)
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
    #define SBE_LITTLE_ENDIAN_ENCODE_16(v) __builtin_bswap16(v)
    #define SBE_LITTLE_ENDIAN_ENCODE_32(v) __builtin_bswap32(v)
    #define SBE_LITTLE_ENDIAN_ENCODE_64(v) __builtin_bswap64(v)
    #define SBE_BIG_ENDIAN_ENCODE_16(v) (v)
    #define SBE_BIG_ENDIAN_ENCODE_32(v) (v)
    #define SBE_BIG_ENDIAN_ENCODE_64(v) (v)
#else
    #error "Byte Ordering of platform not determined. Set __BYTE_ORDER__ manually before including this file."
#endif

#if defined(SBE_NO_BOUNDS_CHECK)
    #define SBE_BOUNDS_CHECK_EXPECT(exp,c) (false)
#elif defined(_MSC_VER)
    #define SBE_BOUNDS_CHECK_EXPECT(exp,c) (exp)
#else
    #define SBE_BOUNDS_CHECK_EXPECT(exp,c) (__builtin_expect(exp,c))
#endif

#if defined(__GNUC__)
    #define SBE_NULLVALUE_INT8 (INT8_MIN)
    #define SBE_NULLVALUE_INT16 (INT16_MIN)
    #define SBE_NULLVALUE_INT32 (INT32_MIN)
    #define SBE_NULLVALUE_INT64 (INT64_MIN)
    #define SBE_NULLVALUE_UINT8 (UINT8_MAX)
    #define SBE_NULLVALUE_UINT16 (UINT16_MAX)
    #define SBE_NULLVALUE_UINT32 (UINT32_MAX)
    #define SBE_NULLVALUE_UINT64 (UINT64_MAX)
#elif defined(_MSC_VER)
    // Visual C++ does not handle minimum integer values properly
    // See: http://msdn.microsoft.com/en-us/library/4kh09110.aspx
    #define SBE_NULLVALUE_INT8 (SCHAR_MIN)
    #define SBE_NULLVALUE_INT16 (SHRT_MIN)
    #define SBE_NULLVALUE_INT32 (LONG_MIN)
    #define SBE_NULLVALUE_INT64 (LLONG_MIN)
    #define SBE_NULLVALUE_UINT8 (UCHAR_MAX)
    #define SBE_NULLVALUE_UINT16 (USHRT_MAX)
    #define SBE_NULLVALUE_UINT32 (ULONG_MAX)
    #define SBE_NULLVALUE_UINT64 (ULLONG_MAX)
#else
    #define SBE_NULLVALUE_INT8 (INT8_MIN)
    #define SBE_NULLVALUE_INT16 (INT16_MIN)
    #define SBE_NULLVALUE_INT32 (INT32_MIN)
    #define SBE_NULLVALUE_INT64 (INT64_MIN)
    #define SBE_NULLVALUE_UINT8 (UINT8_MAX)
    #define SBE_NULLVALUE_UINT16 (UINT16_MAX)
    #define SBE_NULLVALUE_UINT32 (UINT32_MAX)
    #define SBE_NULLVALUE_UINT64 (UINT64_MAX)
#endif

namespace MetaAttribute {

enum Attribute
{
    EPOCH,
    TIME_UNIT,
    SEMANTIC_TYPE
};

}

}

#endif /* _SBE_HPP_ */
<|endoftext|>


<commit_before>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
 * This file is part of the LibreOffice project.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * This file incorporates work covered by the following license notice:
 *
 *   Licensed to the Apache Software Foundation (ASF) under one or more
 *   contributor license agreements. See the NOTICE file distributed
 *   with this work for additional information regarding copyright
 *   ownership. The ASF licenses this file to you under the Apache
 *   License, Version 2.0 (the "License"); you may not use this file
 *   except in compliance with the License. You may obtain a copy of
 *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
 */

#include <vcl/GraphicNativeTransform.hxx>

#include <vcl/gfxlink.hxx>
#include <vcl/graphicfilter.hxx>
#include <com/sun/star/uno/Sequence.hxx>
#include <com/sun/star/beans/XPropertySet.hpp>

#include "jpeg/Exif.hxx"
#include "jpeg/JpegTransform.hxx"

GraphicNativeTransform::GraphicNativeTransform(Graphic& rGraphic) :
    mrGraphic(rGraphic)
{}

GraphicNativeTransform::~GraphicNativeTransform()
{}

bool GraphicNativeTransform::canBeRotated()
{
    GfxLink aLink = mrGraphic.GetLink();

    // Don't allow rotation on animations for now
    if (mrGraphic.IsAnimated())
    {
        return false;
    }

    if (   aLink.GetType() == GFX_LINK_TYPE_NATIVE_JPG
        || aLink.GetType() == GFX_LINK_TYPE_NATIVE_PNG
        || aLink.GetType() == GFX_LINK_TYPE_NATIVE_GIF
        || aLink.GetType() == GFX_LINK_TYPE_NONE)
    {
        return true;
    }

    return false;
}

bool GraphicNativeTransform::rotate(sal_uInt16 aInputRotation)
{
    // Rotation can be between 0 and 3600
    sal_uInt16 aRotation = aInputRotation % 3600;

    if (aRotation == 0)
    {
        return true; // No rotation is needed
    }
    else if (aRotation != 900 && aRotation != 1800 && aRotation != 2700)
    {
        return false;
    }

    GfxLink aLink = mrGraphic.GetLink();
    if ( aLink.GetType() == GFX_LINK_TYPE_NATIVE_JPG )
    {
        return rotateJPEG(aRotation);
    }
    else if ( aLink.GetType() == GFX_LINK_TYPE_NATIVE_PNG )
    {
        return rotateGeneric(aRotation, OUString("png"));
    }
    else if ( aLink.GetType() == GFX_LINK_TYPE_NATIVE_GIF )
    {
        return rotateGeneric(aRotation, OUString("gif"));
    }
    else if ( aLink.GetType() == GFX_LINK_TYPE_NONE )
    {
        return rotateBitmapOnly(aRotation);
    }
    return false;
}

bool GraphicNativeTransform::rotateBitmapOnly(sal_uInt16 aRotation)
{
    if (mrGraphic.IsAnimated())
    {
        return false;
    }

    BitmapEx aBitmap = mrGraphic.GetBitmapEx();
    aBitmap.Rotate(aRotation, COL_BLACK);
    mrGraphic = aBitmap;

    return true;
}

bool GraphicNativeTransform::rotateGeneric(sal_uInt16 aRotation, OUString aType)
{
    // Can't rotate animations yet
    if (mrGraphic.IsAnimated())
    {
        return false;
    }

    SvMemoryStream aStream;

    GraphicFilter& rFilter = GraphicFilter::GetGraphicFilter();

    css::uno::Sequence< css::beans::PropertyValue > aFilterData( 3 );
    aFilterData[ 0 ].Name = "Interlaced";
    aFilterData[ 0 ].Value <<= (sal_Int32) 0;
    aFilterData[ 1 ].Name = "Compression";
    aFilterData[ 1 ].Value <<= (sal_Int32) 9;
    aFilterData[ 2 ].Name = "Quality";
    aFilterData[ 2 ].Value <<= (sal_Int32) 90;

    sal_uInt16 nFilterFormat = rFilter.GetExportFormatNumberForShortName( aType );

    BitmapEx aBitmap = mrGraphic.GetBitmapEx();
    aBitmap.Rotate(aRotation, COL_BLACK);
    rFilter.ExportGraphic( aBitmap, OUString( "none" ), aStream, nFilterFormat, &aFilterData );

    aStream.Seek( STREAM_SEEK_TO_BEGIN );

    Graphic aGraphic;
    rFilter.ImportGraphic( aGraphic, OUString("import"), aStream );

    mrGraphic = aGraphic;
    return true;
}

bool GraphicNativeTransform::rotateJPEG(sal_uInt16 aRotation)
{
    GfxLink aLink = mrGraphic.GetLink();

    SvMemoryStream aSourceStream;
    aSourceStream.Write(aLink.GetData(), aLink.GetDataSize());
    aSourceStream.Seek( STREAM_SEEK_TO_BEGIN );

    Orientation aOrientation = TOP_LEFT;

    Exif exif;
    if ( exif.read(aSourceStream) )
    {
        aOrientation = exif.getOrientation();
    }

    SvMemoryStream aTargetStream;
    JpegTransform tranform(aSourceStream, aTargetStream);
    tranform.setRotate(aRotation);
    tranform.perform();

    aTargetStream.Seek( STREAM_SEEK_TO_BEGIN );

    // Reset orientation in exif if needed
    if ( exif.hasExif() && aOrientation != TOP_LEFT)
    {
        exif.setOrientation(TOP_LEFT);
        exif.write(aTargetStream);
    }

    aTargetStream.Seek( STREAM_SEEK_TO_END );
    sal_uInt32 aBufferSize = aTargetStream.Tell();
    sal_uInt8* pBuffer = new sal_uInt8[ aBufferSize ];

    aTargetStream.Seek( STREAM_SEEK_TO_BEGIN );
    aTargetStream.Read( pBuffer, aBufferSize );

    BitmapEx aBitmap = mrGraphic.GetBitmapEx();
    aBitmap.Rotate(aRotation, COL_BLACK);
    mrGraphic = aBitmap;
    mrGraphic.SetLink( GfxLink( pBuffer, aBufferSize, aLink.GetType(), sal_True ) );

    return true;
}

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
<commit_msg>fdo#66006 Convert to PNG for JPEG that can't be losslessly rotated<commit_after>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
 * This file is part of the LibreOffice project.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * This file incorporates work covered by the following license notice:
 *
 *   Licensed to the Apache Software Foundation (ASF) under one or more
 *   contributor license agreements. See the NOTICE file distributed
 *   with this work for additional information regarding copyright
 *   ownership. The ASF licenses this file to you under the Apache
 *   License, Version 2.0 (the "License"); you may not use this file
 *   except in compliance with the License. You may obtain a copy of
 *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
 */

#include <vcl/GraphicNativeTransform.hxx>

#include <vcl/gfxlink.hxx>
#include <vcl/graphicfilter.hxx>
#include <com/sun/star/uno/Sequence.hxx>
#include <com/sun/star/beans/XPropertySet.hpp>

#include "jpeg/Exif.hxx"
#include "jpeg/JpegTransform.hxx"

GraphicNativeTransform::GraphicNativeTransform(Graphic& rGraphic) :
    mrGraphic(rGraphic)
{}

GraphicNativeTransform::~GraphicNativeTransform()
{}

bool GraphicNativeTransform::canBeRotated()
{
    GfxLink aLink = mrGraphic.GetLink();

    // Don't allow rotation on animations for now
    if (mrGraphic.IsAnimated())
    {
        return false;
    }

    if (   aLink.GetType() == GFX_LINK_TYPE_NATIVE_JPG
        || aLink.GetType() == GFX_LINK_TYPE_NATIVE_PNG
        || aLink.GetType() == GFX_LINK_TYPE_NATIVE_GIF
        || aLink.GetType() == GFX_LINK_TYPE_NONE)
    {
        return true;
    }

    return false;
}

bool GraphicNativeTransform::rotate(sal_uInt16 aInputRotation)
{
    // Rotation can be between 0 and 3600
    sal_uInt16 aRotation = aInputRotation % 3600;

    if (aRotation == 0)
    {
        return true; // No rotation is needed
    }
    else if (aRotation != 900 && aRotation != 1800 && aRotation != 2700)
    {
        return false;
    }

    GfxLink aLink = mrGraphic.GetLink();
    if ( aLink.GetType() == GFX_LINK_TYPE_NATIVE_JPG )
    {
        return rotateJPEG(aRotation);
    }
    else if ( aLink.GetType() == GFX_LINK_TYPE_NATIVE_PNG )
    {
        return rotateGeneric(aRotation, OUString("png"));
    }
    else if ( aLink.GetType() == GFX_LINK_TYPE_NATIVE_GIF )
    {
        return rotateGeneric(aRotation, OUString("gif"));
    }
    else if ( aLink.GetType() == GFX_LINK_TYPE_NONE )
    {
        return rotateBitmapOnly(aRotation);
    }
    return false;
}

bool GraphicNativeTransform::rotateBitmapOnly(sal_uInt16 aRotation)
{
    if (mrGraphic.IsAnimated())
    {
        return false;
    }

    BitmapEx aBitmap = mrGraphic.GetBitmapEx();
    aBitmap.Rotate(aRotation, COL_BLACK);
    mrGraphic = aBitmap;

    return true;
}

bool GraphicNativeTransform::rotateGeneric(sal_uInt16 aRotation, OUString aType)
{
    // Can't rotate animations yet
    if (mrGraphic.IsAnimated())
    {
        return false;
    }

    SvMemoryStream aStream;

    GraphicFilter& rFilter = GraphicFilter::GetGraphicFilter();

    css::uno::Sequence< css::beans::PropertyValue > aFilterData( 3 );
    aFilterData[ 0 ].Name = "Interlaced";
    aFilterData[ 0 ].Value <<= (sal_Int32) 0;
    aFilterData[ 1 ].Name = "Compression";
    aFilterData[ 1 ].Value <<= (sal_Int32) 9;
    aFilterData[ 2 ].Name = "Quality";
    aFilterData[ 2 ].Value <<= (sal_Int32) 90;

    sal_uInt16 nFilterFormat = rFilter.GetExportFormatNumberForShortName( aType );

    BitmapEx aBitmap = mrGraphic.GetBitmapEx();
    aBitmap.Rotate(aRotation, COL_BLACK);
    rFilter.ExportGraphic( aBitmap, OUString( "none" ), aStream, nFilterFormat, &aFilterData );

    aStream.Seek( STREAM_SEEK_TO_BEGIN );

    Graphic aGraphic;
    rFilter.ImportGraphic( aGraphic, OUString("import"), aStream );

    mrGraphic = aGraphic;
    return true;
}

bool GraphicNativeTransform::rotateJPEG(sal_uInt16 aRotation)
{
    BitmapEx aBitmap = mrGraphic.GetBitmapEx();

    if (aBitmap.GetSizePixel().Width()  % 16 != 0 ||
        aBitmap.GetSizePixel().Height() % 16 != 0 )
    {
        rotateGeneric(aRotation, OUString("png"));
    }
    else
    {
        GfxLink aLink = mrGraphic.GetLink();

        SvMemoryStream aSourceStream;
        aSourceStream.Write(aLink.GetData(), aLink.GetDataSize());
        aSourceStream.Seek( STREAM_SEEK_TO_BEGIN );

        Orientation aOrientation = TOP_LEFT;

        Exif exif;
        if ( exif.read(aSourceStream) )
        {
            aOrientation = exif.getOrientation();
        }

        SvMemoryStream aTargetStream;
        JpegTransform tranform(aSourceStream, aTargetStream);
        tranform.setRotate(aRotation);
        tranform.perform();

        aTargetStream.Seek( STREAM_SEEK_TO_BEGIN );

        // Reset orientation in exif if needed
        if ( exif.hasExif() && aOrientation != TOP_LEFT)
        {
            exif.setOrientation(TOP_LEFT);
            exif.write(aTargetStream);
        }

        aTargetStream.Seek( STREAM_SEEK_TO_BEGIN );

        Graphic aGraphic;
        GraphicFilter& rFilter = GraphicFilter::GetGraphicFilter();
        rFilter.ImportGraphic( aGraphic, OUString("import"), aTargetStream );
        mrGraphic = aGraphic;
    }

    return true;
}

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
<|endoftext|>


<commit_before><commit_msg>LOGCXX-10: Override apr-iconv's module locating logic, utf-8 decoding<commit_after><|endoftext|>


<commit_before>#include "Compute.h"
#include "ImageLogger.h"
#include "VideoSource.h"
#include <iostream>
#include <opencv2/imgproc.hpp>

static int redChannel   = 2;
static int greenChannel = 1;
static int blueChannel  = 0;
static int grayChannels = -1;

enum SegMethod {Hough, LSD, Contours};

template<typename T>
static inline T square(T x) {
    return x * x;
}

template<typename T>
static inline T length2(const cv::Vec<T,4>& v) {
    return square(v(0) - v(2)) + square(v(1) - v(3));
}

static bool longer(const cv::Vec4i& v, const cv::Vec4i& w) {
    return length2(v) > length2(w);
}

void OutputData::UseLineForDirection(const cv::Vec4i& line) {
    if (line(1) < line(3)) {
        loX = line(0);
        loY = line(1);
        hiX = line(2);
        hiY = line(3);
    } else {
        hiX = line(0);
        hiY = line(1);
        loX = line(2);
        loY = line(3);
    }

    const int mid = image.cols / 2;
    direction = loY < image.rows/3 && (mid < loX && loX < hiX || mid > loX && loX > hiX)
              ? Turn
              : GoStraight;
}

typedef std::vector<cv::Point> Polygon;
typedef std::vector<Polygon> ContoursT;

void Compute::BackgroundLoop() {
    const SegMethod seg = Contours;

    const int    HoughThreshold = cap->imageWidth * cap->imageHeight / 6500;
    const double HoughMinLineLength = cap->imageHeight / 4.;
    const double HoughMaxGap = cap->imageHeight / 50.;
    const double HoughRho = 2.;

#if 1
    const double CannyThreshold1 =  50.;
    const double CannyThreshold2 = 100.;
    const double HoughTheta = 0.05;
    const int colorChannels = redChannel;
#else
    const double CannyThreshold1 = 100.;
    const double CannyThreshold2 = 200.;
    const double HoughTheta = 0.02;
    const int colorChannels = grayChannels;
#endif

    cv::Ptr<cv::LineSegmentDetector> lsd;
    if (seg == LSD) {
        lsd = cv::createLineSegmentDetector();
    }

    cv::Mat gray8(cap->imageHeight, cap->imageWidth, CV_8UC1), bw;
    cv::Vec4f line;
    static const int grayTo3Ch[3*2] = {0,0, 0,1, 0,2};

    for (int i = 0; ! cap->imagesCaptured.quitting; ++i) {
        cv::Mat inp(cap->imagesCaptured.Dequeue());
        if (! inp.empty()) {

            if (log && i % 10 == 0 && log->imagesToLog.size < log->imagesToLog.Capacity()) {
                log->imagesToLog.Enqueue(inp.clone());
            }

            OutputData out(cap->imageWidth, cap->imageHeight);

            if (seg == Contours) {
                cv::cvtColor(inp, inp, cv::COLOR_BGR2Lab);
                cv::inRange(inp, cv::Scalar(60, 100, 70), cv::Scalar(220, 130, 110), gray8);

                ContoursT contours;
                cv::findContours(gray8, contours, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_SIMPLE);
                for (ContoursT::iterator p = contours.begin(); p != contours.end(); ++p) {
                    Polygon& poly = *p;
                    cv::Rect r = cv::boundingRect(poly);
                    if (r.y < cap->imageHeight/3) {
                        //cv::approxPolyDP(poly, poly, 5., true);
                        cv::fitLine(poly, line, cv::DIST_L2, 0., HoughRho, HoughTheta);
                        cv::Point p1(-9999*line(0) + line(2), -9999*line(1) + line(3)),
                                  p2( 9999*line(0) + line(2),  9999*line(1) + line(3));
                        cv::clipLine(r, p1, p2);
                        out.lines.push_back(cv::Vec4i(p1.x, p1.y, p2.x, p2.y));
                    }
                }
#if 1
                cv::mixChannels(&gray8, 1, &out.image, 1, grayTo3Ch, 3);
#else
                static const int to3Ch[3*2] = {2,0, 2,1, 2,2};
                cv::mixChannels(&inp, 1, &out.image, 1, to3Ch, 3);
                cv::drawContours(out.image, contours, -1, cv::Scalar(0,255,0));
#endif
            }
            else {
                if (colorChannels == grayChannels) {
                    cv::cvtColor(inp, gray8, cv::COLOR_BGR2GRAY);
                } else {
                    const int colorFromTo[1*2] = {colorChannels,0};
                    cv::mixChannels(&inp, 1, &gray8, 1, colorFromTo, 1);
                }

                if (seg == LSD) {
                    lsd->detect(gray8, out.lines);
                } else if (seg == Hough) {
                    cv::Canny(gray8, bw, CannyThreshold1, CannyThreshold2);
                    cv::HoughLinesP(bw, out.lines, HoughRho, HoughTheta, HoughThreshold, HoughMinLineLength, HoughMaxGap);
                }

                cv::mixChannels(&gray8, 1, &out.image, 1, grayTo3Ch, 3);
            }

            std::sort(out.lines.begin(), out.lines.end(), longer);
            out.direction = GoBack;
            for (Lines::iterator l = out.lines.begin();
                 out.direction != Turn && l != out.lines.end() && length2(*l) > square(cap->imageHeight/3);
                 ++l) {
                out.UseLineForDirection(*l);
            }

            SwapOutputData(out);
        }
    }
}

static void* ComputeThread(void* compute) {
    ((Compute*)compute)->BackgroundLoop();
    return 0;
}

void Compute::Start() {
    pthread_create(&thread, NULL, ComputeThread, this);
}

void Compute::Stop() {
    cap->imagesCaptured.Quit();
    pthread_join(thread, NULL);
}

Compute::Compute(VideoSource* c, ImageLogger* lg): cap(c), log(lg) {
    pthread_mutex_init(&dataMutex, NULL);
}

Compute::~Compute() {
    pthread_mutex_destroy(&dataMutex);
}

void Compute::SwapOutputData(OutputData& data) {
    OutputData x = data;

    pthread_mutex_lock(&dataMutex);
    data = outputData;
    outputData = x;
    pthread_mutex_unlock(&dataMutex);
}
<commit_msg>refine the color range from the noir camera<commit_after>#include "Compute.h"
#include "ImageLogger.h"
#include "VideoSource.h"
#include <iostream>
#include <opencv2/imgproc.hpp>

static int redChannel   = 2;
static int greenChannel = 1;
static int blueChannel  = 0;
static int grayChannels = -1;

enum SegMethod {Hough, LSD, Contours};

template<typename T>
static inline T square(T x) {
    return x * x;
}

template<typename T>
static inline T length2(const cv::Vec<T,4>& v) {
    return square(v(0) - v(2)) + square(v(1) - v(3));
}

static bool longer(const cv::Vec4i& v, const cv::Vec4i& w) {
    return length2(v) > length2(w);
}

void OutputData::UseLineForDirection(const cv::Vec4i& line) {
    if (line(1) < line(3)) {
        loX = line(0);
        loY = line(1);
        hiX = line(2);
        hiY = line(3);
    } else {
        hiX = line(0);
        hiY = line(1);
        loX = line(2);
        loY = line(3);
    }

    const int mid = image.cols / 2;
    direction = loY < image.rows/3 && (mid < loX && loX < hiX || mid > loX && loX > hiX)
              ? Turn
              : GoStraight;
}

typedef std::vector<cv::Point> Polygon;
typedef std::vector<Polygon> ContoursT;

void Compute::BackgroundLoop() {
    const SegMethod seg = Contours;

    const int    HoughThreshold = cap->imageWidth * cap->imageHeight / 6500;
    const double HoughMinLineLength = cap->imageHeight / 4.;
    const double HoughMaxGap = cap->imageHeight / 50.;
    const double HoughRho = 2.;

#if 1
    const double CannyThreshold1 =  50.;
    const double CannyThreshold2 = 100.;
    const double HoughTheta = 0.05;
    const int colorChannels = redChannel;
#else
    const double CannyThreshold1 = 100.;
    const double CannyThreshold2 = 200.;
    const double HoughTheta = 0.02;
    const int colorChannels = grayChannels;
#endif

    cv::Ptr<cv::LineSegmentDetector> lsd;
    if (seg == LSD) {
        lsd = cv::createLineSegmentDetector();
    }

    cv::Mat gray8(cap->imageHeight, cap->imageWidth, CV_8UC1), bw;
    cv::Vec4f line;
    static const int grayTo3Ch[3*2] = {0,0, 0,1, 0,2};

    for (int i = 0; ! cap->imagesCaptured.quitting; ++i) {
        cv::Mat inp(cap->imagesCaptured.Dequeue());
        if (! inp.empty()) {

            if (log && i % 10 == 0 && log->imagesToLog.size < log->imagesToLog.Capacity()) {
                log->imagesToLog.Enqueue(inp.clone());
            }

            OutputData out(cap->imageWidth, cap->imageHeight);

            if (seg == Contours) {
                cv::cvtColor(inp, inp, cv::COLOR_BGR2Lab);
                cv::inRange(inp, cv::Scalar(60, 100, 70), cv::Scalar(220, 130, 120), gray8);

                ContoursT contours;
                cv::findContours(gray8, contours, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_SIMPLE);
                for (ContoursT::iterator p = contours.begin(); p != contours.end(); ++p) {
                    Polygon& poly = *p;
                    cv::Rect r = cv::boundingRect(poly);
                    if (r.y < cap->imageHeight/3) {
                        //cv::approxPolyDP(poly, poly, 5., true);
                        cv::fitLine(poly, line, cv::DIST_L2, 0., HoughRho, HoughTheta);
                        cv::Point p1(-9999*line(0) + line(2), -9999*line(1) + line(3)),
                                  p2( 9999*line(0) + line(2),  9999*line(1) + line(3));
                        cv::clipLine(r, p1, p2);
                        out.lines.push_back(cv::Vec4i(p1.x, p1.y, p2.x, p2.y));
                    }
                }
#if 1
                cv::mixChannels(&gray8, 1, &out.image, 1, grayTo3Ch, 3);
#else
                static const int to3Ch[3*2] = {0,0, 0,1, 0,2};
                cv::mixChannels(&inp, 1, &out.image, 1, to3Ch, 3);
                cv::drawContours(out.image, contours, -1, cv::Scalar(0,255,0));
#endif
            }
            else {
                if (colorChannels == grayChannels) {
                    cv::cvtColor(inp, gray8, cv::COLOR_BGR2GRAY);
                } else {
                    const int colorFromTo[1*2] = {colorChannels,0};
                    cv::mixChannels(&inp, 1, &gray8, 1, colorFromTo, 1);
                }

                if (seg == LSD) {
                    lsd->detect(gray8, out.lines);
                } else if (seg == Hough) {
                    cv::Canny(gray8, bw, CannyThreshold1, CannyThreshold2);
                    cv::HoughLinesP(bw, out.lines, HoughRho, HoughTheta, HoughThreshold, HoughMinLineLength, HoughMaxGap);
                }

                cv::mixChannels(&gray8, 1, &out.image, 1, grayTo3Ch, 3);
            }

            std::sort(out.lines.begin(), out.lines.end(), longer);
            out.direction = GoBack;
            for (Lines::iterator l = out.lines.begin();
                 out.direction != Turn && l != out.lines.end() && length2(*l) > square(cap->imageHeight/3);
                 ++l) {
                out.UseLineForDirection(*l);
            }

            SwapOutputData(out);
        }
    }
}

static void* ComputeThread(void* compute) {
    ((Compute*)compute)->BackgroundLoop();
    return 0;
}

void Compute::Start() {
    pthread_create(&thread, NULL, ComputeThread, this);
}

void Compute::Stop() {
    cap->imagesCaptured.Quit();
    pthread_join(thread, NULL);
}

Compute::Compute(VideoSource* c, ImageLogger* lg): cap(c), log(lg) {
    pthread_mutex_init(&dataMutex, NULL);
}

Compute::~Compute() {
    pthread_mutex_destroy(&dataMutex);
}

void Compute::SwapOutputData(OutputData& data) {
    OutputData x = data;

    pthread_mutex_lock(&dataMutex);
    data = outputData;
    outputData = x;
    pthread_mutex_unlock(&dataMutex);
}
<|endoftext|>


<commit_before>/* This file is part of Ingen.
 * Copyright (C) 2007-2009 Dave Robillard <http://drobilla.net>
 *
 * Ingen is free software; you can redistribute it and/or modify it under the
 * terms of the GNU General Public License as published by the Free Software
 * Foundation; either version 2 of the License, or (at your option) any later
 * version.
 *
 * Ingen is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include <glibmm/module.h>
#include <glibmm/miscutils.h>
#include <glibmm/fileutils.h>
#include "raul/log.hpp"
#include "ingen-config.h"
#include "shared/runtime_paths.hpp"
#include "World.hpp"

#define LOG(s) s << "[Module] "

using namespace std;
using namespace Raul;

namespace Ingen {
namespace Shared {

/** Load a dynamic module from the default path.
 *
 * This will check in the directories specified in the environment variable
 * INGEN_MODULE_PATH (typical colon delimited format), then the default module
 * installation directory (ie /usr/local/lib/ingen), in that order.
 *
 * \param name The base name of the module, e.g. "ingen_serialisation"
 */
static SharedPtr<Glib::Module>
load_module(const string& name)
{
	Glib::Module* module = NULL;

	// Search INGEN_MODULE_PATH first
	bool module_path_found;
	string module_path = Glib::getenv("INGEN_MODULE_PATH", module_path_found);
	if (module_path_found) {
		string dir;
		istringstream iss(module_path);
		while (getline(iss, dir, ':')) {
			string filename = Glib::Module::build_path(dir, name);
			if (Glib::file_test(filename, Glib::FILE_TEST_EXISTS)) {
				module = new Glib::Module(filename, Glib::MODULE_BIND_LAZY);
				if (*module) {
					LOG(info) << "Loaded `" <<  name << "' from " << filename << endl;
					return SharedPtr<Glib::Module>(module);
				} else {
					delete module;
					error << Glib::Module::get_last_error() << endl;
				}
			}
		}
	}

	// Try default directory if not found
	module = new Glib::Module(
			Shared::module_path(name),
            Glib::MODULE_BIND_LAZY);

	if (*module) {
		LOG(info) << "Loaded `" <<  name << "' from " << INGEN_MODULE_DIR << endl;
		return SharedPtr<Glib::Module>(module);
	} else if (!module_path_found) {
		LOG(error) << "Unable to find " << name
			<< " (" << Glib::Module::get_last_error() << ")" << endl;
		return SharedPtr<Glib::Module>();
	} else {
		LOG(error) << "Unable to load " << name << " from " << module_path
			<< " (" << Glib::Module::get_last_error() << ")" << endl;
		LOG(error) << "Is Ingen installed?" << endl;
		return SharedPtr<Glib::Module>();
	}
}

/** Load an Ingen module.
 * @return true on success, false on failure
 */
bool
World::load(const char* name)
{
	SharedPtr<Glib::Module> lib = load_module(name);
	Ingen::Shared::Module* (*module_load)() = NULL;
	if (lib && lib->get_symbol("ingen_module_load", (void*&)module_load)) {
		Module* module = module_load();
		module->library = lib;
		module->load(this);
		modules.insert(make_pair(string(name), module));
		return true;
	} else {
		LOG(error) << "Failed to load module " << name << endl;
		return false;
	}
}

/** Unload all loaded Ingen modules.
 */
void
World::unload_all()
{
	modules.clear();
}

/** Get an interface for a remote engine at @a url
 */
SharedPtr<Ingen::Shared::EngineInterface>
World::interface(const std::string& url)
{
	const string scheme = url.substr(0, url.find(":"));
	const InterfaceFactories::const_iterator i = interface_factories.find(scheme);
	if (i == interface_factories.end()) {
		warn << "Unknown URI scheme `'" << scheme << "'" << endl;
		return SharedPtr<Ingen::Shared::EngineInterface>();
	}

	return i->second(this, url);
}

/** Run a script of type @a mime_type at filename @a filename */
bool
World::run(const std::string& mime_type, const std::string& filename)
{
	const ScriptRunners::const_iterator i = script_runners.find(mime_type);
	if (i == script_runners.end()) {
		warn << "Unknown script MIME type `'" << mime_type << "'" << endl;
		return false;
	}

	return i->second(this, filename.c_str());
}

void
World::add_interface_factory(const std::string& scheme, InterfaceFactory factory)
{
	interface_factories.insert(make_pair(scheme, factory));
}

} // namespace Shared
} // namespace Ingen
<commit_msg>Consistent message style.<commit_after>/* This file is part of Ingen.
 * Copyright (C) 2007-2009 Dave Robillard <http://drobilla.net>
 *
 * Ingen is free software; you can redistribute it and/or modify it under the
 * terms of the GNU General Public License as published by the Free Software
 * Foundation; either version 2 of the License, or (at your option) any later
 * version.
 *
 * Ingen is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include <glibmm/module.h>
#include <glibmm/miscutils.h>
#include <glibmm/fileutils.h>
#include "raul/log.hpp"
#include "ingen-config.h"
#include "shared/runtime_paths.hpp"
#include "World.hpp"

#define LOG(s) s << "[Module] "

using namespace std;
using namespace Raul;

namespace Ingen {
namespace Shared {

/** Load a dynamic module from the default path.
 *
 * This will check in the directories specified in the environment variable
 * INGEN_MODULE_PATH (typical colon delimited format), then the default module
 * installation directory (ie /usr/local/lib/ingen), in that order.
 *
 * \param name The base name of the module, e.g. "ingen_serialisation"
 */
static SharedPtr<Glib::Module>
load_module(const string& name)
{
	Glib::Module* module = NULL;

	// Search INGEN_MODULE_PATH first
	bool module_path_found;
	string module_path = Glib::getenv("INGEN_MODULE_PATH", module_path_found);
	if (module_path_found) {
		string dir;
		istringstream iss(module_path);
		while (getline(iss, dir, ':')) {
			string filename = Glib::Module::build_path(dir, name);
			if (Glib::file_test(filename, Glib::FILE_TEST_EXISTS)) {
				module = new Glib::Module(filename, Glib::MODULE_BIND_LAZY);
				if (*module) {
					LOG(info) << "Loaded `" <<  name << "' from " << filename << endl;
					return SharedPtr<Glib::Module>(module);
				} else {
					delete module;
					error << Glib::Module::get_last_error() << endl;
				}
			}
		}
	}

	// Try default directory if not found
	module = new Glib::Module(
			Shared::module_path(name),
            Glib::MODULE_BIND_LAZY);

	if (*module) {
		LOG(info) << "Loaded `" <<  name << "' from " << INGEN_MODULE_DIR << endl;
		return SharedPtr<Glib::Module>(module);
	} else if (!module_path_found) {
		LOG(error) << "Unable to find " << name
			<< " (" << Glib::Module::get_last_error() << ")" << endl;
		return SharedPtr<Glib::Module>();
	} else {
		LOG(error) << "Unable to load " << name << " from " << module_path
			<< " (" << Glib::Module::get_last_error() << ")" << endl;
		LOG(error) << "Is Ingen installed?" << endl;
		return SharedPtr<Glib::Module>();
	}
}

/** Load an Ingen module.
 * @return true on success, false on failure
 */
bool
World::load(const char* name)
{
	SharedPtr<Glib::Module> lib = load_module(name);
	Ingen::Shared::Module* (*module_load)() = NULL;
	if (lib && lib->get_symbol("ingen_module_load", (void*&)module_load)) {
		Module* module = module_load();
		module->library = lib;
		module->load(this);
		modules.insert(make_pair(string(name), module));
		return true;
	} else {
		LOG(error) << "Failed to load module `" << name << "'" << endl;
		return false;
	}
}

/** Unload all loaded Ingen modules.
 */
void
World::unload_all()
{
	modules.clear();
}

/** Get an interface for a remote engine at @a url
 */
SharedPtr<Ingen::Shared::EngineInterface>
World::interface(const std::string& url)
{
	const string scheme = url.substr(0, url.find(":"));
	const InterfaceFactories::const_iterator i = interface_factories.find(scheme);
	if (i == interface_factories.end()) {
		warn << "Unknown URI scheme `" << scheme << "'" << endl;
		return SharedPtr<Ingen::Shared::EngineInterface>();
	}

	return i->second(this, url);
}

/** Run a script of type @a mime_type at filename @a filename */
bool
World::run(const std::string& mime_type, const std::string& filename)
{
	const ScriptRunners::const_iterator i = script_runners.find(mime_type);
	if (i == script_runners.end()) {
		warn << "Unknown script MIME type `" << mime_type << "'" << endl;
		return false;
	}

	return i->second(this, filename.c_str());
}

void
World::add_interface_factory(const std::string& scheme, InterfaceFactory factory)
{
	interface_factories.insert(make_pair(scheme, factory));
}

} // namespace Shared
} // namespace Ingen
<|endoftext|>


<commit_before>// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
 * Ceph - scalable distributed file system
 *
 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
 *
 * This is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License version 2.1, as published by the Free Software
 * Foundation.  See file COPYING.
 *
 */

#include "msg/SimpleMessenger.h"
#include "messages/MMonGetMap.h"
#include "messages/MMonMap.h"
#include "messages/MClientMount.h"

#include "include/nstring.h"

#include "messages/MClientMountAck.h"
#include "messages/MMonSubscribe.h"
#include "messages/MMonSubscribeAck.h"
#include "common/ConfUtils.h"

#include "MonClient.h"
#include "MonMap.h"

#include "config.h"

#define DOUT_SUBSYS monc
#undef dout_prefix
#define dout_prefix *_dout << dbeginl << "monclient" << (hunting ? "(hunting)":"") << ": "

/*
 * build an initial monmap with any known monitor
 * addresses.
 */
int MonClient::build_initial_monmap()
{
  dout(10) << "build_initial_monmap" << dendl;

  // file?
  if (g_conf.monmap) {
    const char *monmap_fn = g_conf.monmap;
    int r = monmap.read(monmap_fn);
    if (r >= 0)
      return 0;
    char buf[80];
    cerr << "unable to read monmap from " << monmap_fn << ": " << strerror_r(errno, buf, sizeof(buf)) << std::endl;
  }

  // -m foo?
  if (g_conf.mon_host) {
    vector<entity_addr_t> addrs;
    if (parse_ip_port_vec(g_conf.mon_host, addrs)) {
      for (unsigned i=0; i<addrs.size(); i++) {
	entity_inst_t inst;
	inst.name = entity_name_t::MON(i);
	inst.addr = addrs[i];
	monmap.add_mon(inst);
      }
      return 0;
    }
    cerr << "unable to parse addrs in '" << g_conf.mon_host << "'" << std::endl;
  }

  // config file?
  ConfFile a(g_conf.conf);
  ConfFile b("ceph.conf");
  ConfFile *c = 0;

  if (a.parse())
    c = &a;
  else if (b.parse())
    c = &b;
  if (c) {
    static string monstr;
    for (int i=0; i<15; i++) {
      char monname[20];
      char *val = 0;
      sprintf(monname, "mon%d", i);
      c->read(monname, "mon addr", &val, 0);
      if (!val || !val[0])
	break;

      entity_inst_t inst;
      if (!parse_ip_port(val, inst.addr)) {
	cerr << "unable to parse conf's addr for " << monname << " (" << val << ")" << std::endl;
	continue;
      }
      inst.name = entity_name_t::MON(monmap.mon_inst.size());
      monmap.add_mon(inst);
    }
    if (monmap.size())
      return 0;
    cerr << "unable to find any monitors in conf" << std::endl;
    return -EINVAL;
  }

  cerr << "please specify monitors via -m monaddr or -c ceph.conf" << std::endl;
  return -ENOENT;
}

int MonClient::get_monmap()
{
  dout(10) << "get_monmap" << dendl;
  Mutex::Locker l(monc_lock);

  _sub_want("monmap", monmap.get_epoch());
  want_monmap = true;
  if (cur_mon < 0)
    _reopen_session();

  while (want_monmap)
    map_cond.Wait(monc_lock);

  dout(10) << "get_monmap done" << dendl;
  return 0;
}

int MonClient::get_monmap_privately()
{
  dout(10) << "get_monmap_privately" << dendl;
  Mutex::Locker l(monc_lock);

  SimpleMessenger *rank = NULL;
  bool temp_msgr = false;
  if (!messenger) {
    rank = new SimpleMessenger;
    messenger = rank->register_entity(entity_name_t::CLIENT(-1));
    messenger->add_dispatcher_head(this);
    rank->start(true);  // do not daemonize!
    temp_msgr = true;
  }

  int attempt = 10;
  int i = 0;
  srand(getpid());

  dout(10) << "have " << monmap.epoch << dendl;

  while (monmap.epoch == 0) {
    i = rand() % monmap.mon_inst.size();
    dout(10) << "querying " << monmap.mon_inst[i] << dendl;
    messenger->send_message(new MMonGetMap, monmap.mon_inst[i]);

    if (--attempt == 0)
      break;

    utime_t interval(1, 0);
    map_cond.WaitInterval(monc_lock, interval);
  }

  if (temp_msgr) {
    messenger->shutdown();
    rank->wait();
    messenger->destroy();
    messenger = 0;
  }

  hunting = true;  // reset this to true!

  if (monmap.epoch)
    return 0;
  return -1;
}

bool MonClient::ms_dispatch(Message *m)
{
  if (my_addr == entity_addr_t())
    my_addr = messenger->get_myaddr();

  switch (m->get_type()) {
  case CEPH_MSG_MON_MAP:
    handle_monmap((MMonMap*)m);
    return true;

  case CEPH_MSG_CLIENT_MOUNT_ACK:
    handle_mount_ack((MClientMountAck*)m);
    return true;

  case CEPH_MSG_MON_SUBSCRIBE_ACK:
    handle_subscribe_ack((MMonSubscribeAck*)m);
    return true;
  }

  return false;
}

void MonClient::handle_monmap(MMonMap *m)
{
  dout(10) << "handle_monmap " << *m << dendl;
  monc_lock.Lock();

  _finish_hunting();

  bufferlist::iterator p = m->monmapbl.begin();
  ::decode(monmap, p);

  _sub_got("monmap", monmap.get_epoch());

  map_cond.Signal();
  want_monmap = false;

  monc_lock.Unlock();
  delete m;
}

// -------------------
// MOUNT

void MonClient::_send_mount()
{
  dout(10) << "_send_mount" << dendl;
  _send_mon_message(new MClientMount);
  mount_started = g_clock.now();
}

void MonClient::init()
{
  dout(10) << "init" << dendl;
  messenger->add_dispatcher_head(this);

  Mutex::Locker l(monc_lock);
  timer.add_event_after(10.0, new C_Tick(this));
}

void MonClient::shutdown()
{
  timer.cancel_all_events();
}

int MonClient::mount(double mount_timeout)
{
  Mutex::Locker lock(monc_lock);

  if (clientid >= 0) {
    dout(5) << "already mounted" << dendl;;
    return 0;
  }

  // only first mounter does the work
  _sub_want("monmap", monmap.get_epoch());
  mounting++;
  if (mounting == 1) {
    if (cur_mon < 0)
      _reopen_session();
    else
      _send_mount();
  }
  while (clientid < 0 && !mount_err)
    mount_cond.Wait(monc_lock);
  mounting--;

  if (clientid >= 0) {
    dout(5) << "mount success, client" << clientid << dendl;

    _sub_want("monmap", monmap.get_epoch());
  }

  return mount_err;
}

void MonClient::handle_mount_ack(MClientMountAck* m)
{
  dout(10) << "handle_mount_ack " << *m << dendl;

  _finish_hunting();

  if (m->result) {
    mount_err = m->result;
    dout(0) << "mount error " << m->result << " (" << m->result_msg << ")" << dendl;
  } else {
    // monmap
    bufferlist::iterator p = m->monmap_bl.begin();
    ::decode(monmap, p);

    clientid = m->client;
    messenger->set_myname(entity_name_t::CLIENT(m->client.v));
  }

  mount_cond.SignalAll();

  delete m;
}

// ---------

void MonClient::_send_mon_message(Message *m)
{
  assert(cur_mon >= 0);
  messenger->send_message(m, monmap.mon_inst[cur_mon]);
}

void MonClient::_pick_new_mon()
{
  if (cur_mon >= 0)
    messenger->mark_down(monmap.get_inst(cur_mon).addr);
  cur_mon = rand() % monmap.size();
  dout(10) << "_pick_new_mon picked mon" << cur_mon << dendl;
}

void MonClient::_reopen_session()
{
  dout(10) << "_reopen_session" << dendl;
  _pick_new_mon();
  if (mounting)
    _send_mount();
  if (!sub_have.empty())
    _renew_subs();
  if (!mounting && sub_have.empty()) {
    _send_mon_message(new MMonGetMap);
  }
}

bool MonClient::ms_handle_reset(Connection *con)
{
  Mutex::Locker lock(monc_lock);

  if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) {
    if (con->get_peer_addr() != monmap.get_inst(cur_mon).addr) {
      dout(10) << "ms_handle_reset stray mon " << con->get_peer_addr() << dendl;
      return true;
    } else {
      dout(10) << "ms_handle_reset current mon " << con->get_peer_addr() << dendl;
      if (hunting)
	return true;

      dout(0) << "hunting for new mon" << dendl;
      hunting = true;
      _reopen_session();
    }
  }
  return false;
}

void MonClient::_finish_hunting()
{
  if (hunting) {
    dout(0) << "found mon" << cur_mon << dendl;
    hunting = false;
  }
}

void MonClient::tick()
{
  dout(10) << "tick" << dendl;

  if (hunting) {
    dout(0) << "continuing hunt" << dendl;
    _reopen_session();

  } else {
    // just renew as needed
    utime_t now = g_clock.now();
    if (now > sub_renew_after)
      _renew_subs();

    messenger->send_keepalive(monmap.mon_inst[cur_mon]);
  }

  timer.add_event_after(10.0, new C_Tick(this));
}

// ---------

void MonClient::_renew_subs()
{
  if (sub_have.empty()) {
    dout(10) << "renew_subs - empty" << dendl;
    return;
  }

  dout(10) << "renew_subs" << dendl;
  if (cur_mon < 0)
    _reopen_session();
  else {
    if (sub_renew_sent == utime_t())
      sub_renew_sent = g_clock.now();

    MMonSubscribe *m = new MMonSubscribe;
    m->what = sub_have;
    _send_mon_message(m);
  }
}

void MonClient::handle_subscribe_ack(MMonSubscribeAck *m)
{
  _finish_hunting();

  if (sub_renew_sent != utime_t()) {
    sub_renew_after = sub_renew_sent;
    sub_renew_after += m->interval / 2.0;
    dout(10) << "handle_subscribe_ack sent " << sub_renew_sent << " renew after " << sub_renew_after << dendl;
    sub_renew_sent = utime_t();
  } else {
    dout(10) << "handle_subscribe_ack sent " << sub_renew_sent << ", ignoring" << dendl;
  }
}

<commit_msg>monc: cleanup stray sub_want()<commit_after>// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
 * Ceph - scalable distributed file system
 *
 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
 *
 * This is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License version 2.1, as published by the Free Software
 * Foundation.  See file COPYING.
 *
 */

#include "msg/SimpleMessenger.h"
#include "messages/MMonGetMap.h"
#include "messages/MMonMap.h"
#include "messages/MClientMount.h"

#include "include/nstring.h"

#include "messages/MClientMountAck.h"
#include "messages/MMonSubscribe.h"
#include "messages/MMonSubscribeAck.h"
#include "common/ConfUtils.h"

#include "MonClient.h"
#include "MonMap.h"

#include "config.h"

#define DOUT_SUBSYS monc
#undef dout_prefix
#define dout_prefix *_dout << dbeginl << "monclient" << (hunting ? "(hunting)":"") << ": "

/*
 * build an initial monmap with any known monitor
 * addresses.
 */
int MonClient::build_initial_monmap()
{
  dout(10) << "build_initial_monmap" << dendl;

  // file?
  if (g_conf.monmap) {
    const char *monmap_fn = g_conf.monmap;
    int r = monmap.read(monmap_fn);
    if (r >= 0)
      return 0;
    char buf[80];
    cerr << "unable to read monmap from " << monmap_fn << ": " << strerror_r(errno, buf, sizeof(buf)) << std::endl;
  }

  // -m foo?
  if (g_conf.mon_host) {
    vector<entity_addr_t> addrs;
    if (parse_ip_port_vec(g_conf.mon_host, addrs)) {
      for (unsigned i=0; i<addrs.size(); i++) {
	entity_inst_t inst;
	inst.name = entity_name_t::MON(i);
	inst.addr = addrs[i];
	monmap.add_mon(inst);
      }
      return 0;
    }
    cerr << "unable to parse addrs in '" << g_conf.mon_host << "'" << std::endl;
  }

  // config file?
  ConfFile a(g_conf.conf);
  ConfFile b("ceph.conf");
  ConfFile *c = 0;

  if (a.parse())
    c = &a;
  else if (b.parse())
    c = &b;
  if (c) {
    static string monstr;
    for (int i=0; i<15; i++) {
      char monname[20];
      char *val = 0;
      sprintf(monname, "mon%d", i);
      c->read(monname, "mon addr", &val, 0);
      if (!val || !val[0])
	break;

      entity_inst_t inst;
      if (!parse_ip_port(val, inst.addr)) {
	cerr << "unable to parse conf's addr for " << monname << " (" << val << ")" << std::endl;
	continue;
      }
      inst.name = entity_name_t::MON(monmap.mon_inst.size());
      monmap.add_mon(inst);
    }
    if (monmap.size())
      return 0;
    cerr << "unable to find any monitors in conf" << std::endl;
    return -EINVAL;
  }

  cerr << "please specify monitors via -m monaddr or -c ceph.conf" << std::endl;
  return -ENOENT;
}

int MonClient::get_monmap()
{
  dout(10) << "get_monmap" << dendl;
  Mutex::Locker l(monc_lock);

  _sub_want("monmap", monmap.get_epoch());
  want_monmap = true;
  if (cur_mon < 0)
    _reopen_session();

  while (want_monmap)
    map_cond.Wait(monc_lock);

  dout(10) << "get_monmap done" << dendl;
  return 0;
}

int MonClient::get_monmap_privately()
{
  dout(10) << "get_monmap_privately" << dendl;
  Mutex::Locker l(monc_lock);

  SimpleMessenger *rank = NULL;
  bool temp_msgr = false;
  if (!messenger) {
    rank = new SimpleMessenger;
    messenger = rank->register_entity(entity_name_t::CLIENT(-1));
    messenger->add_dispatcher_head(this);
    rank->start(true);  // do not daemonize!
    temp_msgr = true;
  }

  int attempt = 10;
  int i = 0;
  srand(getpid());

  dout(10) << "have " << monmap.epoch << dendl;

  while (monmap.epoch == 0) {
    i = rand() % monmap.mon_inst.size();
    dout(10) << "querying " << monmap.mon_inst[i] << dendl;
    messenger->send_message(new MMonGetMap, monmap.mon_inst[i]);

    if (--attempt == 0)
      break;

    utime_t interval(1, 0);
    map_cond.WaitInterval(monc_lock, interval);
  }

  if (temp_msgr) {
    messenger->shutdown();
    rank->wait();
    messenger->destroy();
    messenger = 0;
  }

  hunting = true;  // reset this to true!

  if (monmap.epoch)
    return 0;
  return -1;
}

bool MonClient::ms_dispatch(Message *m)
{
  if (my_addr == entity_addr_t())
    my_addr = messenger->get_myaddr();

  switch (m->get_type()) {
  case CEPH_MSG_MON_MAP:
    handle_monmap((MMonMap*)m);
    return true;

  case CEPH_MSG_CLIENT_MOUNT_ACK:
    handle_mount_ack((MClientMountAck*)m);
    return true;

  case CEPH_MSG_MON_SUBSCRIBE_ACK:
    handle_subscribe_ack((MMonSubscribeAck*)m);
    return true;
  }

  return false;
}

void MonClient::handle_monmap(MMonMap *m)
{
  dout(10) << "handle_monmap " << *m << dendl;
  monc_lock.Lock();

  _finish_hunting();

  bufferlist::iterator p = m->monmapbl.begin();
  ::decode(monmap, p);

  _sub_got("monmap", monmap.get_epoch());

  map_cond.Signal();
  want_monmap = false;

  monc_lock.Unlock();
  delete m;
}

// -------------------
// MOUNT

void MonClient::_send_mount()
{
  dout(10) << "_send_mount" << dendl;
  _send_mon_message(new MClientMount);
  mount_started = g_clock.now();
}

void MonClient::init()
{
  dout(10) << "init" << dendl;
  messenger->add_dispatcher_head(this);

  Mutex::Locker l(monc_lock);
  timer.add_event_after(10.0, new C_Tick(this));
}

void MonClient::shutdown()
{
  timer.cancel_all_events();
}

int MonClient::mount(double mount_timeout)
{
  Mutex::Locker lock(monc_lock);

  if (clientid >= 0) {
    dout(5) << "already mounted" << dendl;;
    return 0;
  }

  // only first mounter does the work
  _sub_want("monmap", monmap.get_epoch());
  mounting++;
  if (mounting == 1) {
    if (cur_mon < 0)
      _reopen_session();
    else
      _send_mount();
  }
  while (clientid < 0 && !mount_err)
    mount_cond.Wait(monc_lock);
  mounting--;

  if (clientid >= 0)
    dout(5) << "mount success, client" << clientid << dendl;

  return mount_err;
}

void MonClient::handle_mount_ack(MClientMountAck* m)
{
  dout(10) << "handle_mount_ack " << *m << dendl;

  _finish_hunting();

  if (m->result) {
    mount_err = m->result;
    dout(0) << "mount error " << m->result << " (" << m->result_msg << ")" << dendl;
  } else {
    // monmap
    bufferlist::iterator p = m->monmap_bl.begin();
    ::decode(monmap, p);

    clientid = m->client;
    messenger->set_myname(entity_name_t::CLIENT(m->client.v));
  }

  mount_cond.SignalAll();

  delete m;
}

// ---------

void MonClient::_send_mon_message(Message *m)
{
  assert(cur_mon >= 0);
  messenger->send_message(m, monmap.mon_inst[cur_mon]);
}

void MonClient::_pick_new_mon()
{
  if (cur_mon >= 0)
    messenger->mark_down(monmap.get_inst(cur_mon).addr);
  cur_mon = rand() % monmap.size();
  dout(10) << "_pick_new_mon picked mon" << cur_mon << dendl;
}

void MonClient::_reopen_session()
{
  dout(10) << "_reopen_session" << dendl;
  _pick_new_mon();
  if (mounting)
    _send_mount();
  if (!sub_have.empty())
    _renew_subs();
  if (!mounting && sub_have.empty()) {
    _send_mon_message(new MMonGetMap);
  }
}

bool MonClient::ms_handle_reset(Connection *con)
{
  Mutex::Locker lock(monc_lock);

  if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) {
    if (con->get_peer_addr() != monmap.get_inst(cur_mon).addr) {
      dout(10) << "ms_handle_reset stray mon " << con->get_peer_addr() << dendl;
      return true;
    } else {
      dout(10) << "ms_handle_reset current mon " << con->get_peer_addr() << dendl;
      if (hunting)
	return true;

      dout(0) << "hunting for new mon" << dendl;
      hunting = true;
      _reopen_session();
    }
  }
  return false;
}

void MonClient::_finish_hunting()
{
  if (hunting) {
    dout(0) << "found mon" << cur_mon << dendl;
    hunting = false;
  }
}

void MonClient::tick()
{
  dout(10) << "tick" << dendl;

  if (hunting) {
    dout(0) << "continuing hunt" << dendl;
    _reopen_session();

  } else {
    // just renew as needed
    utime_t now = g_clock.now();
    if (now > sub_renew_after)
      _renew_subs();

    messenger->send_keepalive(monmap.mon_inst[cur_mon]);
  }

  timer.add_event_after(10.0, new C_Tick(this));
}

// ---------

void MonClient::_renew_subs()
{
  if (sub_have.empty()) {
    dout(10) << "renew_subs - empty" << dendl;
    return;
  }

  dout(10) << "renew_subs" << dendl;
  if (cur_mon < 0)
    _reopen_session();
  else {
    if (sub_renew_sent == utime_t())
      sub_renew_sent = g_clock.now();

    MMonSubscribe *m = new MMonSubscribe;
    m->what = sub_have;
    _send_mon_message(m);
  }
}

void MonClient::handle_subscribe_ack(MMonSubscribeAck *m)
{
  _finish_hunting();

  if (sub_renew_sent != utime_t()) {
    sub_renew_after = sub_renew_sent;
    sub_renew_after += m->interval / 2.0;
    dout(10) << "handle_subscribe_ack sent " << sub_renew_sent << " renew after " << sub_renew_after << dendl;
    sub_renew_sent = utime_t();
  } else {
    dout(10) << "handle_subscribe_ack sent " << sub_renew_sent << ", ignoring" << dendl;
  }
}

<|endoftext|>


<commit_before><commit_msg>fixed comment<commit_after><|endoftext|>


<commit_before>// Copyright © 2014 German Neuroinformatics Node (G-Node)
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted under the terms of the BSD License. See
// LICENSE file in the root of the Project.
//
// Author: Christian Kellner <kellner@bio.lmu.de>

#include <cstdio>
#include <nix.hpp>
#include <nix/NDArray.hpp>
#include <queue>
#include <random>
#include <type_traits>
#include <iostream>
#include <mutex>
#include <condition_variable>
#include <thread>
#include <stdexcept>

/* ************************************ */

class Stopwatch {

public:
    typedef std::chrono::high_resolution_clock::time_point time_point_t;
    typedef std::chrono::high_resolution_clock clock_t;

    Stopwatch() : t_start(clock_t::now()) { };

    ssize_t ms() {
        time_point_t t_end = clock_t::now();
        ssize_t count = std::chrono::duration_cast<std::chrono::milliseconds>(t_end - t_start).count();
        return count;
    }

private:
    time_point_t t_start;
};

/* ************************************ */

class RndGenBase {
public:
    RndGenBase() : rd(), rd_gen(rd()) { };

protected:
    std::random_device rd;
    std::mt19937 rd_gen;
};

template<typename T, typename Enable = void>
class RndGen { };

template<typename T>
class RndGen<T, typename std::enable_if<std::is_floating_point<T>::value >::type> : RndGenBase {
public:
    RndGen() : dis(-1024.0, +1024.0) { };

    T operator()(void) {
        return dis(rd_gen);
    };

private:
    std::uniform_real_distribution<T> dis;
};

template<typename T>
class RndGen<T, typename std::enable_if<std::is_integral<T>::value >::type> : RndGenBase {
public:
    RndGen() : dis(-1024, +1024) { };

    T operator()(void) {
        return dis(rd_gen);
    };

private:
    std::uniform_real_distribution<T> dis;
};

/* ************************************ */

template<typename T>
class BlockGenerator {
public:

    BlockGenerator(size_t blocksize, size_t bufsize) : blocksize(blocksize), bufsize(bufsize) { };

    std::vector<T> make_block() {
        std::vector<T> block(blocksize);
        std::generate(block.begin(), block.end(), std::ref(rnd_gen));
        return block;
    }

    std::vector<T> next_block() {
        std::unique_lock<std::mutex> lock(mtx);

        if (queue.empty()) {
            cnd.wait(lock, [this]{ return !queue.empty(); });
        }

        std::vector<T> x(std::move(queue.front()));
        queue.pop();
        cnd.notify_all();
        return x;
    }

    void worker_thread() {
        while (do_run) {
            std::unique_lock<std::mutex> lock(mtx);
            std::vector<T> block = make_block();

            if (queue.size() > bufsize) {
                //wait until there is room
                cnd.wait(lock, [this]{ return !do_run || queue.size() < bufsize;});
                if (!do_run) {
                    return;
                }
            }

            queue.push(std::move(block));
            cnd.notify_all();
        }
    }

    void start_worker() {
        workers.emplace_back(&BlockGenerator::worker_thread, this);
    }

    ~BlockGenerator() {
        do_run = false;
        cnd.notify_all();
        for (auto &w : workers) {
            w.join();
        }
    }

    double speed_test() {

        Stopwatch sw;

        size_t N = 100;
        size_t iterations = 0;

        do {
            Stopwatch inner;

            for (size_t i = 0; i < N; i++) {
                std::vector<double> block = next_block();
                iterations++;
            }

            if (inner.ms() < 100) {
                N *= 2;
            }

        } while (sw.ms() < 1000);

        ssize_t count = sw.ms();
        return iterations * blocksize * sizeof(T) * (1000.0/count) / (1024.0*1024.0);
    }

private:
    bool do_run = true;
    size_t blocksize;
    size_t bufsize;
    RndGen<T> rnd_gen;
    std::mutex mtx;
    std::condition_variable cnd;
    std::queue<std::vector<T>> queue;
    std::vector<std::thread> workers;
};

class Config {

public:
    Config(nix::DataType data_type, const nix::NDSize &blocksize)
            : data_type(data_type), block_size(blocksize) {

        sdim = find_single_dim();
        shape = blocksize;
        shape[sdim] = 0;

        make_name();
    };

    size_t find_single_dim() {
        size_t sdim;
        bool have_rdim = false;
        for (size_t i = 0; i < block_size.size(); i ++) {
            if (block_size[i] == 1) {
                sdim = i;
                have_rdim = true;
            }
        }

        if (!have_rdim) {
            throw std::runtime_error("Could not find singelton dimension");
        }
        return sdim;
    }

    nix::DataType dtype() const { return data_type; }
    const nix::NDSize& size() const { return block_size; }
    const nix::NDSize& extend() const { return shape; }
    size_t singleton_dimension() const { return sdim; }
    const std::string & name() const { return my_name; };

private:
    void make_name() {
        std::stringstream s;

        s << data_type << "@{ ";
        for (auto x : block_size) {
            s << x << " ";
        }
        s << "}";

        my_name = s.str();
    }

private:
    const nix::DataType data_type;
    const nix::NDSize block_size;

    size_t        sdim;
    nix::NDSize   shape;

    std::string   my_name;
};

class Benchmark {
public:
    Benchmark(const Config &cfg) : config(cfg) { }
    const Config & cfg() const { return config; }

    nix::DataArray openDataArray(nix::Block block) const {
        const std::string &cfg_name = config.name();
        std::vector<nix::DataArray> v = block.dataArrays(nix::util::NameFilter<nix::DataArray>(cfg_name));
        if (v.empty()) {
            return block.createDataArray(cfg_name, "nix.test.da", config.dtype(), config.extend());
        } else {
            return v[0];
        }
    }

    virtual ~Benchmark() { }

    virtual double speed_in_mbs() {
        return count * config.size().nelms() * nix::data_type_to_size(config.dtype()) *
                (1000.0/millis) / (1024 * 1024);
    }

    virtual double speed_in_nps() {
        return count * config.size().nelms() * (1000.0/millis);
    }

    template<typename F>
    ssize_t time_it(F func) {
        Stopwatch watch;
        func();
        return watch.ms();
    }

    virtual void run(nix::Block block) = 0;
    virtual std::string id() = 0;

protected:
    const Config config;
    size_t       count;
    double       millis;
};

class GeneratorBenchmark : public Benchmark {
public:
    GeneratorBenchmark(const Config &cfg)
            : Benchmark(cfg) {

    };

    void run(nix::Block block) override {
        switch (config.dtype()) {

            case nix::DataType::Double:
                test_speed<double>();
                break;

            default:
                throw std::runtime_error("Unsupported DataType!");
        }
    }

    template<typename T>
    void test_speed() {
        size_t nelms = config.size().nelms();
        BlockGenerator<T> generator(nelms, 10);
        generator.start_worker();
        speed = generator.speed_test();
    }

    double speed_in_mbs() override {
        return speed;
    }

    std::string id() override {
        return "P";
    }

private:
    double speed;
};

class WriteBenchmark : public Benchmark {

public:
    WriteBenchmark(const Config &cfg)
            : Benchmark(cfg) {
    };

    void run(nix::Block block) override {
        nix::DataArray da = openDataArray(block);

        switch (config.dtype()) {

            case nix::DataType::Double:
                do_write_test<double>(da);
                break;

            default:
                throw std::runtime_error("Unsupported DataType!");
        }
    }

    std::string id() override {
        return "W";
    }

private:
    template<typename T>
    void do_write_test(nix::DataArray da) {
        size_t nelms = config.size().nelms();
        BlockGenerator<T> generator(nelms, 10);
        generator.start_worker();

        size_t N = 100;
        size_t iterations = 0;

        nix::NDSize pos = {0, 0};
        Stopwatch sw;
        ssize_t ms = 0;
        do {
            Stopwatch inner;

            for (size_t i = 0; i < N; i++) {
                std::vector<T> block = generator.next_block();
                da.dataExtent(config.size() + pos);
                da.setData(config.dtype(), block.data(), config.size(), pos);
                pos[config.singleton_dimension()] += 1;
                iterations++;
            }

            if (inner.ms() < 100) {
                N *= 2;
            }

        } while ((ms = sw.ms()) < 3*1000);

        this->count = iterations;
        this->millis = ms;
    }
};

class ReadBenchmark : public Benchmark {

public:
    ReadBenchmark(const Config &cfg)
            : Benchmark(cfg) {
    };

    void test_read_io(nix::DataArray da) {

        nix::NDArray array(config.dtype(), config.size());
        nix::NDSize extend = da.dataExtent();
        size_t N = extend[config.singleton_dimension()];

        nix::NDSize pos = {0, 0};

        ssize_t ms = time_it([this, &da, &N, &pos, &array] {
            for(size_t i = 0; i < N; i++) {
                da.getData(config.dtype(), array.data(), config.size(), pos);
                pos[config.singleton_dimension()] += 1;
            }
        });

        this->count = N;
        this->millis = ms;
    }

    void run(nix::Block block) override {
        nix::DataArray da = openDataArray(block);
        test_read_io(da);
    }

    std::string id() override {
        return "R";
    }
};

class ReadPolyBenchmark : public ReadBenchmark {

public:
    ReadPolyBenchmark(const Config &cfg)
            : ReadBenchmark(cfg) {
    };

    void run(nix::Block block) override {
        nix::DataArray da = openDataArray(block);
        da.polynomCoefficients({3, 4, 5, 6});
        test_read_io(da);
    }

    std::string id() override {
        return "P";
    }
};

/* ************************************ */

static std::vector<Config> make_configs() {

    std::vector<Config> configs;

    configs.emplace_back(nix::DataType::Double, nix::NDSize{2048, 1});
    configs.emplace_back(nix::DataType::Double, nix::NDSize{1, 2048});

    return configs;
}

int main(int argc, char **argv)
{
    nix::File fd = nix::File::open("iospeed.h5", nix::FileMode::Overwrite);
    nix::Block block = fd.createBlock("speed", "nix.test");

    std::vector<Config> configs = make_configs();
    std::vector<Benchmark *> marks;

    std::cout << "Performing generators tests..." << std::endl;
    for (const Config &cfg : configs) {
        GeneratorBenchmark *benchmark = new GeneratorBenchmark(cfg);
        benchmark->run(block);
        marks.push_back(benchmark);
    }

    std::cout << "Performing write tests..." << std::endl;
    for (const Config &cfg : configs) {
        WriteBenchmark *benchmark = new WriteBenchmark(cfg);
        benchmark->run(block);
        marks.push_back(benchmark);
    }

    std::cout << "Performing read tests..." << std::endl;
    for (const Config &cfg : configs) {
        ReadBenchmark *benchmark = new ReadBenchmark(cfg);
        benchmark->run(block);
        marks.push_back(benchmark);
    }

    std::cout << "Performing read (poly) tests..." << std::endl;
    for (const Config &cfg : configs) {
        ReadPolyBenchmark *benchmark = new ReadPolyBenchmark(cfg);
        benchmark->run(block);
        marks.push_back(benchmark);
    }

    std::cout << " === Reports ===" << std::endl;
    std::cout.precision(5);
    std::cout.unsetf (std::ios::floatfield);
    for (Benchmark *mark : marks) {
        std::cout << mark->cfg().name() << ", " << mark->id() << ", "
                << mark->speed_in_mbs() << " MB/s, "
                << mark->speed_in_nps() << " N/s" << std::endl;
        delete mark;
    }

    return 0;
}
<commit_msg>[test] Benchmark: Generator: add number per sec<commit_after>// Copyright © 2014 German Neuroinformatics Node (G-Node)
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted under the terms of the BSD License. See
// LICENSE file in the root of the Project.
//
// Author: Christian Kellner <kellner@bio.lmu.de>

#include <cstdio>
#include <nix.hpp>
#include <nix/NDArray.hpp>
#include <queue>
#include <random>
#include <type_traits>
#include <iostream>
#include <mutex>
#include <condition_variable>
#include <thread>
#include <stdexcept>

/* ************************************ */

class Stopwatch {

public:
    typedef std::chrono::high_resolution_clock::time_point time_point_t;
    typedef std::chrono::high_resolution_clock clock_t;

    Stopwatch() : t_start(clock_t::now()) { };

    ssize_t ms() {
        time_point_t t_end = clock_t::now();
        ssize_t count = std::chrono::duration_cast<std::chrono::milliseconds>(t_end - t_start).count();
        return count;
    }

private:
    time_point_t t_start;
};

/* ************************************ */

class RndGenBase {
public:
    RndGenBase() : rd(), rd_gen(rd()) { };

protected:
    std::random_device rd;
    std::mt19937 rd_gen;
};

template<typename T, typename Enable = void>
class RndGen { };

template<typename T>
class RndGen<T, typename std::enable_if<std::is_floating_point<T>::value >::type> : RndGenBase {
public:
    RndGen() : dis(-1024.0, +1024.0) { };

    T operator()(void) {
        return dis(rd_gen);
    };

private:
    std::uniform_real_distribution<T> dis;
};

template<typename T>
class RndGen<T, typename std::enable_if<std::is_integral<T>::value >::type> : RndGenBase {
public:
    RndGen() : dis(-1024, +1024) { };

    T operator()(void) {
        return dis(rd_gen);
    };

private:
    std::uniform_real_distribution<T> dis;
};

/* ************************************ */

template<typename T>
class BlockGenerator {
public:

    BlockGenerator(size_t blocksize, size_t bufsize) : blocksize(blocksize), bufsize(bufsize) { };

    std::vector<T> make_block() {
        std::vector<T> block(blocksize);
        std::generate(block.begin(), block.end(), std::ref(rnd_gen));
        return block;
    }

    std::vector<T> next_block() {
        std::unique_lock<std::mutex> lock(mtx);

        if (queue.empty()) {
            cnd.wait(lock, [this]{ return !queue.empty(); });
        }

        std::vector<T> x(std::move(queue.front()));
        queue.pop();
        cnd.notify_all();
        return x;
    }

    void worker_thread() {
        while (do_run) {
            std::unique_lock<std::mutex> lock(mtx);
            std::vector<T> block = make_block();

            if (queue.size() > bufsize) {
                //wait until there is room
                cnd.wait(lock, [this]{ return !do_run || queue.size() < bufsize;});
                if (!do_run) {
                    return;
                }
            }

            queue.push(std::move(block));
            cnd.notify_all();
        }
    }

    void start_worker() {
        workers.emplace_back(&BlockGenerator::worker_thread, this);
    }

    ~BlockGenerator() {
        do_run = false;
        cnd.notify_all();
        for (auto &w : workers) {
            w.join();
        }
    }

    double speed_test(size_t &iterations) {

        Stopwatch sw;

        size_t N = 100;
        iterations = 0;

        do {
            Stopwatch inner;

            for (size_t i = 0; i < N; i++) {
                std::vector<double> block = next_block();
                iterations++;
            }

            if (inner.ms() < 100) {
                N *= 2;
            }

        } while (sw.ms() < 1000);

        ssize_t count = sw.ms();
        return count;
    }

private:
    bool do_run = true;
    size_t blocksize;
    size_t bufsize;
    RndGen<T> rnd_gen;
    std::mutex mtx;
    std::condition_variable cnd;
    std::queue<std::vector<T>> queue;
    std::vector<std::thread> workers;
};

class Config {

public:
    Config(nix::DataType data_type, const nix::NDSize &blocksize)
            : data_type(data_type), block_size(blocksize) {

        sdim = find_single_dim();
        shape = blocksize;
        shape[sdim] = 0;

        make_name();
    };

    size_t find_single_dim() {
        size_t sdim;
        bool have_rdim = false;
        for (size_t i = 0; i < block_size.size(); i ++) {
            if (block_size[i] == 1) {
                sdim = i;
                have_rdim = true;
            }
        }

        if (!have_rdim) {
            throw std::runtime_error("Could not find singelton dimension");
        }
        return sdim;
    }

    nix::DataType dtype() const { return data_type; }
    const nix::NDSize& size() const { return block_size; }
    const nix::NDSize& extend() const { return shape; }
    size_t singleton_dimension() const { return sdim; }
    const std::string & name() const { return my_name; };

private:
    void make_name() {
        std::stringstream s;

        s << data_type << "@{ ";
        for (auto x : block_size) {
            s << x << " ";
        }
        s << "}";

        my_name = s.str();
    }

private:
    const nix::DataType data_type;
    const nix::NDSize block_size;

    size_t        sdim;
    nix::NDSize   shape;

    std::string   my_name;
};

class Benchmark {
public:
    Benchmark(const Config &cfg) : config(cfg) { }
    const Config & cfg() const { return config; }

    nix::DataArray openDataArray(nix::Block block) const {
        const std::string &cfg_name = config.name();
        std::vector<nix::DataArray> v = block.dataArrays(nix::util::NameFilter<nix::DataArray>(cfg_name));
        if (v.empty()) {
            return block.createDataArray(cfg_name, "nix.test.da", config.dtype(), config.extend());
        } else {
            return v[0];
        }
    }

    virtual ~Benchmark() { }

    virtual double speed_in_mbs() {
        return count * config.size().nelms() * nix::data_type_to_size(config.dtype()) *
                (1000.0/millis) / (1024 * 1024);
    }

    virtual double speed_in_nps() {
        return count * config.size().nelms() * (1000.0/millis);
    }

    template<typename F>
    ssize_t time_it(F func) {
        Stopwatch watch;
        func();
        return watch.ms();
    }

    virtual void run(nix::Block block) = 0;
    virtual std::string id() = 0;

protected:
    const Config config;
    size_t       count;
    double       millis;
};

class GeneratorBenchmark : public Benchmark {
public:
    GeneratorBenchmark(const Config &cfg)
            : Benchmark(cfg) {

    };

    void run(nix::Block block) override {
        switch (config.dtype()) {

            case nix::DataType::Double:
                test_speed<double>();
                break;

            default:
                throw std::runtime_error("Unsupported DataType!");
        }
    }

    template<typename T>
    void test_speed() {
        size_t nelms = config.size().nelms();
        BlockGenerator<T> generator(nelms, 10);
        generator.start_worker();
        this->millis = generator.speed_test(this->count);
    }

    std::string id() override {
        return "P";
    }
};

class WriteBenchmark : public Benchmark {

public:
    WriteBenchmark(const Config &cfg)
            : Benchmark(cfg) {
    };

    void run(nix::Block block) override {
        nix::DataArray da = openDataArray(block);

        switch (config.dtype()) {

            case nix::DataType::Double:
                do_write_test<double>(da);
                break;

            default:
                throw std::runtime_error("Unsupported DataType!");
        }
    }

    std::string id() override {
        return "W";
    }

private:
    template<typename T>
    void do_write_test(nix::DataArray da) {
        size_t nelms = config.size().nelms();
        BlockGenerator<T> generator(nelms, 10);
        generator.start_worker();

        size_t N = 100;
        size_t iterations = 0;

        nix::NDSize pos = {0, 0};
        Stopwatch sw;
        ssize_t ms = 0;
        do {
            Stopwatch inner;

            for (size_t i = 0; i < N; i++) {
                std::vector<T> block = generator.next_block();
                da.dataExtent(config.size() + pos);
                da.setData(config.dtype(), block.data(), config.size(), pos);
                pos[config.singleton_dimension()] += 1;
                iterations++;
            }

            if (inner.ms() < 100) {
                N *= 2;
            }

        } while ((ms = sw.ms()) < 3*1000);

        this->count = iterations;
        this->millis = ms;
    }
};

class ReadBenchmark : public Benchmark {

public:
    ReadBenchmark(const Config &cfg)
            : Benchmark(cfg) {
    };

    void test_read_io(nix::DataArray da) {

        nix::NDArray array(config.dtype(), config.size());
        nix::NDSize extend = da.dataExtent();
        size_t N = extend[config.singleton_dimension()];

        nix::NDSize pos = {0, 0};

        ssize_t ms = time_it([this, &da, &N, &pos, &array] {
            for(size_t i = 0; i < N; i++) {
                da.getData(config.dtype(), array.data(), config.size(), pos);
                pos[config.singleton_dimension()] += 1;
            }
        });

        this->count = N;
        this->millis = ms;
    }

    void run(nix::Block block) override {
        nix::DataArray da = openDataArray(block);
        test_read_io(da);
    }

    std::string id() override {
        return "R";
    }
};

class ReadPolyBenchmark : public ReadBenchmark {

public:
    ReadPolyBenchmark(const Config &cfg)
            : ReadBenchmark(cfg) {
    };

    void run(nix::Block block) override {
        nix::DataArray da = openDataArray(block);
        da.polynomCoefficients({3, 4, 5, 6});
        test_read_io(da);
    }

    std::string id() override {
        return "P";
    }
};

/* ************************************ */

static std::vector<Config> make_configs() {

    std::vector<Config> configs;

    configs.emplace_back(nix::DataType::Double, nix::NDSize{2048, 1});
    configs.emplace_back(nix::DataType::Double, nix::NDSize{1, 2048});

    return configs;
}

int main(int argc, char **argv)
{
    nix::File fd = nix::File::open("iospeed.h5", nix::FileMode::Overwrite);
    nix::Block block = fd.createBlock("speed", "nix.test");

    std::vector<Config> configs = make_configs();
    std::vector<Benchmark *> marks;

    std::cout << "Performing generators tests..." << std::endl;
    for (const Config &cfg : configs) {
        GeneratorBenchmark *benchmark = new GeneratorBenchmark(cfg);
        benchmark->run(block);
        marks.push_back(benchmark);
    }

    std::cout << "Performing write tests..." << std::endl;
    for (const Config &cfg : configs) {
        WriteBenchmark *benchmark = new WriteBenchmark(cfg);
        benchmark->run(block);
        marks.push_back(benchmark);
    }

    std::cout << "Performing read tests..." << std::endl;
    for (const Config &cfg : configs) {
        ReadBenchmark *benchmark = new ReadBenchmark(cfg);
        benchmark->run(block);
        marks.push_back(benchmark);
    }

    std::cout << "Performing read (poly) tests..." << std::endl;
    for (const Config &cfg : configs) {
        ReadPolyBenchmark *benchmark = new ReadPolyBenchmark(cfg);
        benchmark->run(block);
        marks.push_back(benchmark);
    }

    std::cout << " === Reports ===" << std::endl;
    std::cout.precision(5);
    std::cout.unsetf (std::ios::floatfield);
    for (Benchmark *mark : marks) {
        std::cout << mark->cfg().name() << ", " << mark->id() << ", "
                << mark->speed_in_mbs() << " MB/s, "
                << mark->speed_in_nps() << " N/s" << std::endl;
        delete mark;
    }

    return 0;
}
<|endoftext|>