<commit_before>/****************************************************************************
Copyright (c) 2014 Naomasa Matsubayashi

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
****************************************************************************/

#ifndef RAPIDMP_PARSER_HPP
#define RAPIDMP_PARSER_HPP

#include <cstdint>
#include <algorithm>
#include <boost/foreach.hpp>
#include <boost/variant.hpp>
#include <boost/variant/recursive_variant.hpp>
#include <boost/spirit/include/karma.hpp>

#include <rapidmp/config.hpp>
#include <rapidmp/type.hpp>
#include <rapidmp/exceptions.hpp>

namespace rapidmp {
  template< typename Length >
  UMP_FUNCTION decltype( boost::spirit::karma::byte_ )
  get_karma_rule_for_the_integer(
    typename boost::enable_if< boost::mpl::equal_to< Length, boost::mpl::size_t< 1 > > >::type* = 0
  ) {
    return boost::spirit::karma::byte_;
  }

  template< typename Length >
  UMP_FUNCTION decltype( boost::spirit::karma::big_word )
  get_karma_rule_for_the_integer(
    typename boost::enable_if< boost::mpl::equal_to< Length, boost::mpl::size_t< 2 > > >::type* = 0
  ) {
    return boost::spirit::karma::big_word;
  }

  template< typename Length >
  UMP_FUNCTION decltype( boost::spirit::karma::big_dword )
  get_karma_rule_for_the_integer(
    typename boost::enable_if< boost::mpl::equal_to< Length, boost::mpl::size_t< 4 > > >::type* = 0
  ) {
    return boost::spirit::karma::big_dword;
  }

  template< typename Length >
  UMP_FUNCTION decltype( boost::spirit::karma::big_qword )
  get_karma_rule_for_the_integer(
    typename boost::enable_if< boost::mpl::equal_to< Length, boost::mpl::size_t< 8 > > >::type* = 0
  ) {
    return boost::spirit::karma::big_qword;
  }

  template< typename Length, typename Iterator >
  UMP_FUNCTION void generate_uint_in_the_length(
    Iterator &output, uint64_t value
  ) {
    namespace karma = boost::spirit::karma;
    if( !karma::generate( output, get_karma_rule_for_the_integer< Length >(), value ) )
      throw overflow();
  }

  template< typename Iterator >
  UMP_FUNCTION void generate_uint( Iterator &output, uint64_t value ) {
    if( value < 0x80ull ) {
      *output = value;
      ++output;
    }
    else if( value < 0x100ull ) {
      *output = '\xcc';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 1 > >( output, value );
    }
    else if( value < 0x10000ull ) {
      *output = '\xcd' ;
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 2 > >( output, value );
    }
    else if( value < 0x100000000ull ) {
      *output = '\xce';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 4 > >( output, value );
    }
    else {
      *output = '\xcf';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 8 > >( output, value );
    }
  }

  template< typename Length, typename Iterator >
  UMP_FUNCTION void generate_int_in_the_length(
    Iterator &output, int64_t value
  ) {
    namespace karma = boost::spirit::karma;
    if( !karma::generate( output, get_karma_rule_for_the_integer< Length >(), static_cast< typename boost::uint_t< Length::value * 8 >::exact >( value ) ) )
      throw overflow();
  }
  template< typename Iterator >
  UMP_FUNCTION void generate_int( Iterator &output, int64_t value ) {
    if( value <= -0x100000000ll || value >= 0x100000000ll ) {
      *output = '\xd3';
      ++output;
      generate_int_in_the_length< boost::mpl::size_t< 8 > >( output, value );
    }
    else if( value <= -0x10000ll || value >= 0x10000ll ) {
      *output = '\xd2';
      ++output;
      generate_int_in_the_length< boost::mpl::size_t< 4 > >( output, value );
    }
    else if( value <= -0x100ll || value >= 0x100ll ) {
      *output = '\xd1';
      ++output;
      generate_int_in_the_length< boost::mpl::size_t< 2 > >( output, value );
    }
    else if( value >= 0  || value < -32 ) {
      *output = '\xd0';
      ++output;
      generate_int_in_the_length< boost::mpl::size_t< 1 > >( output, value );
    }
    else {
      *output = value;
      ++output;
    }
  }

  template< typename InputIterator, typename OutputIterator >
  UMP_FUNCTION void generate_string( OutputIterator &output, const string< InputIterator > &value ) {
    const size_t length = boost::distance( value.range );
    if( length < 32u ) {
      *output = '\xa0' + length;
      ++output;
    }
    else if( length < 0x100u ) {
      *output = '\xd9';
      ++output;
      generate_int_in_the_length< boost::mpl::size_t< 1 > >( output, length );
    }
    else if( length < 0x10000u ) {
      *output = '\xda';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 2 > >( output, length );
    }
    else if( length < 0x100000000u ) {
      *output = '\xdb';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 4 > >( output, length );
    }
    else throw too_big();
    std::copy( boost::begin( value.range ), boost::end( value.range ), output );
  }

  template< typename InputIterator, typename OutputIterator >
  UMP_FUNCTION void generate_binary( OutputIterator &output, const binary< InputIterator > &value ) {
    const size_t length = boost::distance( value.range );
    if( length < 0x100u ) {
      *output = '\xc4';
      ++output;
      generate_int_in_the_length< boost::mpl::size_t< 1 > >( output, length );
    }
    else if( length < 0x10000u ) {
      *output = '\xc5';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 2 > >( output, length );
    }
    else if( length < 0x100000000u ) {
      *output = '\xc6';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 4 > >( output, length );
    }
    else throw too_big();
    std::copy( boost::begin( value.range ), boost::end( value.range ), output );
  }

  template< typename InputIterator, typename OutputIterator >
  UMP_FUNCTION void generate_extension( OutputIterator &output, const extension< InputIterator > &value ) {
    const size_t length = boost::distance( value.range );
    if( length == 1 ) {
      *output = '\xd4';
      ++output;
    }
    else if( length == 2 ) {
      *output = '\xd5';
      ++output;
    }
    else if( length == 4 ) {
      *output = '\xd6';
      ++output;
    }
    else if( length == 8 ) {
      *output = '\xd7';
      ++output;
    }
    else if( length == 16 ) {
      *output = '\xd8';
      ++output;
    }
    else if( length < 0x100u ) {
      *output = '\xc7';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 1 > >( output, length );
    }
    else if( length < 0x10000u ) {
      *output = '\xc8';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 2 > >( output, length );
    }
    else if( length < 0x100000000u ) {
      *output = '\xc9';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 4 > >( output, length );
    }
    else throw too_big();
    generate_uint_in_the_length< boost::mpl::size_t< 1 > >( output, value.type );
    std::copy( boost::begin( value.range ), boost::end( value.range ), output );
  }

  template< typename InputIterator, typename OutputIterator >
  UMP_FUNCTION void generate_array( OutputIterator &output, const typename array_type< InputIterator >::type &value );

  template< typename InputIterator, typename OutputIterator >
  UMP_FUNCTION void generate_struct( OutputIterator &output, const typename struct_type< InputIterator >::type &value );

  template< typename OutputIterator >
  UMP_FUNCTION void generate_float32( OutputIterator &output, float value ) {
    *output = '\xca';
    ++output;
    namespace karma = boost::spirit::karma;
    if( !karma::generate( output, karma::big_bin_float, value ) )
      throw invalid_float();
  }

  template< typename OutputIterator >
  UMP_FUNCTION void generate_float64( OutputIterator &output, double value ) {
    *output = '\xcb';
    ++output;
    namespace karma = boost::spirit::karma;
    if( !karma::generate( output, karma::big_bin_double, value ) )
      throw invalid_float();
  }

  template< typename OutputIterator >
  UMP_FUNCTION void generate_none( OutputIterator &output ) {
    *output = '\xc0';
    ++output;
  }

  template< typename OutputIterator >
  UMP_FUNCTION void generate_bool( OutputIterator &output, bool value ) {
    *output = value ? '\xc3' : '\xc2';
    ++output;
  }

  template< typename InputIterator, typename OutputIterator >
  class generate_object_visitor : public boost::static_visitor<void> {
  public:
    UMP_FUNCTION generate_object_visitor( OutputIterator &output_ ) : output( output_ ) {}
    UMP_FUNCTION void operator()( uint64_t value ) {
      generate_uint( output, value );
    }
    UMP_FUNCTION void operator()( int64_t value ) {
      generate_int( output, value );
    }
    UMP_FUNCTION void operator()( float value ) {
      generate_float32( output, value );
    }
    UMP_FUNCTION void operator()( double value ) {
      generate_float64( output, value );
    }
    UMP_FUNCTION void operator()( const none_type & ) {
      generate_none( output );
    }
    UMP_FUNCTION void operator()( bool value ) {
      generate_bool( output, value );
    }
    UMP_FUNCTION void operator()( const string< InputIterator > &value ) {
      generate_string( output, value );
    }
    UMP_FUNCTION void operator()( const binary< InputIterator > &value ) {
      generate_binary( output, value );
    }
    UMP_FUNCTION void operator()( const extension< InputIterator > &value ) {
      generate_extension( output, value );
    }
    UMP_FUNCTION void operator()( const typename array_type< InputIterator >::type &value ) {
      generate_array< InputIterator, OutputIterator >( output, value );
    }
    UMP_FUNCTION void operator()( const typename struct_type< InputIterator >::type &value ) {
      generate_struct< InputIterator, OutputIterator >( output, value );
    }
  private:
    OutputIterator &output;
  };

  template< typename ObjectType >
  struct get_input_iterator {
    typedef typename boost::mpl::at_c< typename ObjectType::types, 7 >::type::range_type::iterator type;
  };

  template< typename ObjectType, typename OutputIterator >
  UMP_FUNCTION void generate_object( OutputIterator &output, const ObjectType &value ) {
    generate_object_visitor< typename get_input_iterator< ObjectType >::type, OutputIterator > generator( output );
    apply_visitor( generator, value );
  }

  template< typename InputIterator, typename OutputIterator >
  UMP_FUNCTION void generate_array( OutputIterator &output, const typename array_type< InputIterator >::type &value ) {
    const size_t length = boost::distance( value );
    if( length < 16u ) {
      *output = '\x90' + length;
      ++output;
    }
    else if( length < 0x10000u ) {
      *output = '\xdc';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 2 > >( output, length );
    }
    else if( length < 0x100000000u ) {
      *output = '\xdd';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 4 > >( output, length );
    }
    else throw too_big();
    generate_object_visitor< InputIterator, OutputIterator > generator( output );
    BOOST_FOREACH( const auto &elem, value )
      apply_visitor( generator, elem );
  }

  template< typename InputIterator, typename OutputIterator >
  UMP_FUNCTION void generate_struct( OutputIterator &output, const typename struct_type< InputIterator >::type &value ) {
    const size_t length = boost::distance( value );
    if( length < 16u ) {
      *output = '\x80' + length;
      ++output;
    }
    else if( length < 0x10000u ) {
      *output = '\xde';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 2 > >( output, length );
    }
    else if( length < 0x100000000u ) {
      *output = '\xdf';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 4 > >( output, length );
    }
    else throw too_big();
    generate_object_visitor< InputIterator, OutputIterator > generator( output );
    BOOST_FOREACH( const auto &elem, value ) {
      apply_visitor( generator, elem.first );
      apply_visitor( generator, elem.second );
    }
  }
}

#endif

<commit_msg>Fixed duplicated include guard<commit_after>/****************************************************************************
Copyright (c) 2014 Naomasa Matsubayashi

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
****************************************************************************/

#ifndef RAPIDMP_GENERATOR_HPP
#define RAPIDMP_GENERATOR_HPP

#include <cstdint>
#include <algorithm>
#include <boost/foreach.hpp>
#include <boost/variant.hpp>
#include <boost/variant/recursive_variant.hpp>
#include <boost/spirit/include/karma.hpp>

#include <rapidmp/config.hpp>
#include <rapidmp/type.hpp>
#include <rapidmp/exceptions.hpp>

namespace rapidmp {
  template< typename Length >
  UMP_FUNCTION decltype( boost::spirit::karma::byte_ )
  get_karma_rule_for_the_integer(
    typename boost::enable_if< boost::mpl::equal_to< Length, boost::mpl::size_t< 1 > > >::type* = 0
  ) {
    return boost::spirit::karma::byte_;
  }

  template< typename Length >
  UMP_FUNCTION decltype( boost::spirit::karma::big_word )
  get_karma_rule_for_the_integer(
    typename boost::enable_if< boost::mpl::equal_to< Length, boost::mpl::size_t< 2 > > >::type* = 0
  ) {
    return boost::spirit::karma::big_word;
  }

  template< typename Length >
  UMP_FUNCTION decltype( boost::spirit::karma::big_dword )
  get_karma_rule_for_the_integer(
    typename boost::enable_if< boost::mpl::equal_to< Length, boost::mpl::size_t< 4 > > >::type* = 0
  ) {
    return boost::spirit::karma::big_dword;
  }

  template< typename Length >
  UMP_FUNCTION decltype( boost::spirit::karma::big_qword )
  get_karma_rule_for_the_integer(
    typename boost::enable_if< boost::mpl::equal_to< Length, boost::mpl::size_t< 8 > > >::type* = 0
  ) {
    return boost::spirit::karma::big_qword;
  }

  template< typename Length, typename Iterator >
  UMP_FUNCTION void generate_uint_in_the_length(
    Iterator &output, uint64_t value
  ) {
    namespace karma = boost::spirit::karma;
    if( !karma::generate( output, get_karma_rule_for_the_integer< Length >(), value ) )
      throw overflow();
  }

  template< typename Iterator >
  UMP_FUNCTION void generate_uint( Iterator &output, uint64_t value ) {
    if( value < 0x80ull ) {
      *output = value;
      ++output;
    }
    else if( value < 0x100ull ) {
      *output = '\xcc';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 1 > >( output, value );
    }
    else if( value < 0x10000ull ) {
      *output = '\xcd' ;
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 2 > >( output, value );
    }
    else if( value < 0x100000000ull ) {
      *output = '\xce';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 4 > >( output, value );
    }
    else {
      *output = '\xcf';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 8 > >( output, value );
    }
  }

  template< typename Length, typename Iterator >
  UMP_FUNCTION void generate_int_in_the_length(
    Iterator &output, int64_t value
  ) {
    namespace karma = boost::spirit::karma;
    if( !karma::generate( output, get_karma_rule_for_the_integer< Length >(), static_cast< typename boost::uint_t< Length::value * 8 >::exact >( value ) ) )
      throw overflow();
  }
  template< typename Iterator >
  UMP_FUNCTION void generate_int( Iterator &output, int64_t value ) {
    if( value <= -0x100000000ll || value >= 0x100000000ll ) {
      *output = '\xd3';
      ++output;
      generate_int_in_the_length< boost::mpl::size_t< 8 > >( output, value );
    }
    else if( value <= -0x10000ll || value >= 0x10000ll ) {
      *output = '\xd2';
      ++output;
      generate_int_in_the_length< boost::mpl::size_t< 4 > >( output, value );
    }
    else if( value <= -0x100ll || value >= 0x100ll ) {
      *output = '\xd1';
      ++output;
      generate_int_in_the_length< boost::mpl::size_t< 2 > >( output, value );
    }
    else if( value >= 0  || value < -32 ) {
      *output = '\xd0';
      ++output;
      generate_int_in_the_length< boost::mpl::size_t< 1 > >( output, value );
    }
    else {
      *output = value;
      ++output;
    }
  }

  template< typename InputIterator, typename OutputIterator >
  UMP_FUNCTION void generate_string( OutputIterator &output, const string< InputIterator > &value ) {
    const size_t length = boost::distance( value.range );
    if( length < 32u ) {
      *output = '\xa0' + length;
      ++output;
    }
    else if( length < 0x100u ) {
      *output = '\xd9';
      ++output;
      generate_int_in_the_length< boost::mpl::size_t< 1 > >( output, length );
    }
    else if( length < 0x10000u ) {
      *output = '\xda';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 2 > >( output, length );
    }
    else if( length < 0x100000000u ) {
      *output = '\xdb';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 4 > >( output, length );
    }
    else throw too_big();
    std::copy( boost::begin( value.range ), boost::end( value.range ), output );
  }

  template< typename InputIterator, typename OutputIterator >
  UMP_FUNCTION void generate_binary( OutputIterator &output, const binary< InputIterator > &value ) {
    const size_t length = boost::distance( value.range );
    if( length < 0x100u ) {
      *output = '\xc4';
      ++output;
      generate_int_in_the_length< boost::mpl::size_t< 1 > >( output, length );
    }
    else if( length < 0x10000u ) {
      *output = '\xc5';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 2 > >( output, length );
    }
    else if( length < 0x100000000u ) {
      *output = '\xc6';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 4 > >( output, length );
    }
    else throw too_big();
    std::copy( boost::begin( value.range ), boost::end( value.range ), output );
  }

  template< typename InputIterator, typename OutputIterator >
  UMP_FUNCTION void generate_extension( OutputIterator &output, const extension< InputIterator > &value ) {
    const size_t length = boost::distance( value.range );
    if( length == 1 ) {
      *output = '\xd4';
      ++output;
    }
    else if( length == 2 ) {
      *output = '\xd5';
      ++output;
    }
    else if( length == 4 ) {
      *output = '\xd6';
      ++output;
    }
    else if( length == 8 ) {
      *output = '\xd7';
      ++output;
    }
    else if( length == 16 ) {
      *output = '\xd8';
      ++output;
    }
    else if( length < 0x100u ) {
      *output = '\xc7';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 1 > >( output, length );
    }
    else if( length < 0x10000u ) {
      *output = '\xc8';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 2 > >( output, length );
    }
    else if( length < 0x100000000u ) {
      *output = '\xc9';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 4 > >( output, length );
    }
    else throw too_big();
    generate_uint_in_the_length< boost::mpl::size_t< 1 > >( output, value.type );
    std::copy( boost::begin( value.range ), boost::end( value.range ), output );
  }

  template< typename InputIterator, typename OutputIterator >
  UMP_FUNCTION void generate_array( OutputIterator &output, const typename array_type< InputIterator >::type &value );

  template< typename InputIterator, typename OutputIterator >
  UMP_FUNCTION void generate_struct( OutputIterator &output, const typename struct_type< InputIterator >::type &value );

  template< typename OutputIterator >
  UMP_FUNCTION void generate_float32( OutputIterator &output, float value ) {
    *output = '\xca';
    ++output;
    namespace karma = boost::spirit::karma;
    if( !karma::generate( output, karma::big_bin_float, value ) )
      throw invalid_float();
  }

  template< typename OutputIterator >
  UMP_FUNCTION void generate_float64( OutputIterator &output, double value ) {
    *output = '\xcb';
    ++output;
    namespace karma = boost::spirit::karma;
    if( !karma::generate( output, karma::big_bin_double, value ) )
      throw invalid_float();
  }

  template< typename OutputIterator >
  UMP_FUNCTION void generate_none( OutputIterator &output ) {
    *output = '\xc0';
    ++output;
  }

  template< typename OutputIterator >
  UMP_FUNCTION void generate_bool( OutputIterator &output, bool value ) {
    *output = value ? '\xc3' : '\xc2';
    ++output;
  }

  template< typename InputIterator, typename OutputIterator >
  class generate_object_visitor : public boost::static_visitor<void> {
  public:
    UMP_FUNCTION generate_object_visitor( OutputIterator &output_ ) : output( output_ ) {}
    UMP_FUNCTION void operator()( uint64_t value ) {
      generate_uint( output, value );
    }
    UMP_FUNCTION void operator()( int64_t value ) {
      generate_int( output, value );
    }
    UMP_FUNCTION void operator()( float value ) {
      generate_float32( output, value );
    }
    UMP_FUNCTION void operator()( double value ) {
      generate_float64( output, value );
    }
    UMP_FUNCTION void operator()( const none_type & ) {
      generate_none( output );
    }
    UMP_FUNCTION void operator()( bool value ) {
      generate_bool( output, value );
    }
    UMP_FUNCTION void operator()( const string< InputIterator > &value ) {
      generate_string( output, value );
    }
    UMP_FUNCTION void operator()( const binary< InputIterator > &value ) {
      generate_binary( output, value );
    }
    UMP_FUNCTION void operator()( const extension< InputIterator > &value ) {
      generate_extension( output, value );
    }
    UMP_FUNCTION void operator()( const typename array_type< InputIterator >::type &value ) {
      generate_array< InputIterator, OutputIterator >( output, value );
    }
    UMP_FUNCTION void operator()( const typename struct_type< InputIterator >::type &value ) {
      generate_struct< InputIterator, OutputIterator >( output, value );
    }
  private:
    OutputIterator &output;
  };

  template< typename ObjectType >
  struct get_input_iterator {
    typedef typename boost::mpl::at_c< typename ObjectType::types, 7 >::type::range_type::iterator type;
  };

  template< typename ObjectType, typename OutputIterator >
  UMP_FUNCTION void generate_object( OutputIterator &output, const ObjectType &value ) {
    generate_object_visitor< typename get_input_iterator< ObjectType >::type, OutputIterator > generator( output );
    apply_visitor( generator, value );
  }

  template< typename InputIterator, typename OutputIterator >
  UMP_FUNCTION void generate_array( OutputIterator &output, const typename array_type< InputIterator >::type &value ) {
    const size_t length = boost::distance( value );
    if( length < 16u ) {
      *output = '\x90' + length;
      ++output;
    }
    else if( length < 0x10000u ) {
      *output = '\xdc';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 2 > >( output, length );
    }
    else if( length < 0x100000000u ) {
      *output = '\xdd';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 4 > >( output, length );
    }
    else throw too_big();
    generate_object_visitor< InputIterator, OutputIterator > generator( output );
    BOOST_FOREACH( const auto &elem, value )
      apply_visitor( generator, elem );
  }

  template< typename InputIterator, typename OutputIterator >
  UMP_FUNCTION void generate_struct( OutputIterator &output, const typename struct_type< InputIterator >::type &value ) {
    const size_t length = boost::distance( value );
    if( length < 16u ) {
      *output = '\x80' + length;
      ++output;
    }
    else if( length < 0x10000u ) {
      *output = '\xde';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 2 > >( output, length );
    }
    else if( length < 0x100000000u ) {
      *output = '\xdf';
      ++output;
      generate_uint_in_the_length< boost::mpl::size_t< 4 > >( output, length );
    }
    else throw too_big();
    generate_object_visitor< InputIterator, OutputIterator > generator( output );
    BOOST_FOREACH( const auto &elem, value ) {
      apply_visitor( generator, elem.first );
      apply_visitor( generator, elem.second );
    }
  }
}

#endif

<|endoftext|>


<commit_before>#ifndef _TOOLBOX_CURL_EASY_HPP_
#define _TOOLBOX_CURL_EASY_HPP_

#include <curl/curl.h>

namespace toolbox
{
namespace curl
{

namespace options
{

template<int E, typename T>
struct option_def
{
	using type_t = T;
	static constexpr auto option = static_cast<CURLoption>(E);
	const T &value;

	option_def(const T &value) :
		value (value) {}
};

using verbose = option_def<CURLOPT_VERBOSE, long>;
using url = option_def<CURLOPT_URL, const char*>;
using postfields = option_def<CURLOPT_POSTFIELDS, const char*>;

} // namespace options

class easy
{
public:
	easy() :
		curl_ (curl_easy_init())
	{
	}

	easy(easy &&other) :
		curl_(other.curl_)
	{
		other.curl_ = nullptr;
	}

	void operator=(easy &&other)
	{
		curl_ = other.curl_;
		other.curl_ = nullptr;
	}

	easy(const easy &) = delete;
	void operator=(const easy &) = delete;

	template<typename T>
	auto set(const T &value)
	{
		return curl_easy_setopt(curl_, T::option, value.value);
	}

	CURLcode perform()
	{
		return curl_easy_perform(curl_);
	}

	~easy()
	{
		if (curl_)
			curl_easy_cleanup (curl_);
	}

private:
	CURL* curl_ = nullptr;
};

} // namespace curl
} // namespace toolbox

#endif
<commit_msg>curl: erasy: writefunction and writedata def<commit_after>#ifndef _TOOLBOX_CURL_EASY_HPP_
#define _TOOLBOX_CURL_EASY_HPP_

#include <curl/curl.h>

namespace toolbox
{
namespace curl
{

namespace options
{

template<int E, typename T>
struct option_def
{
	using type_t = T;
	static constexpr auto option = static_cast<CURLoption>(E);
	const T &value;

	option_def(const T &value) :
		value (value) {}
};

using verbose = option_def<CURLOPT_VERBOSE, long>;
using url = option_def<CURLOPT_URL, const char*>;
using postfields = option_def<CURLOPT_POSTFIELDS, const char*>;
using writefunction = option_def<CURLOPT_WRITEFUNCTION, size_t(*)(char *, size_t, size_t, void *)>;
using writedata = option_def<CURLOPT_WRITEDATA, void *>;

} // namespace options

class easy
{
public:
	easy() :
		curl_ (curl_easy_init())
	{
	}

	easy(easy &&other) :
		curl_(other.curl_)
	{
		other.curl_ = nullptr;
	}

	void operator=(easy &&other)
	{
		curl_ = other.curl_;
		other.curl_ = nullptr;
	}

	easy(const easy &) = delete;
	void operator=(const easy &) = delete;

	template<typename T>
	auto set(const T &value)
	{
		return curl_easy_setopt(curl_, T::option, value.value);
	}

	CURLcode perform()
	{
		return curl_easy_perform(curl_);
	}

	~easy()
	{
		if (curl_)
			curl_easy_cleanup (curl_);
	}

private:
	CURL* curl_ = nullptr;
};

} // namespace curl
} // namespace toolbox

#endif
<|endoftext|>


<commit_before>#ifndef _INCLUDED_INPUT_HPP
#define _INCLUDED_INPUT_HPP

#include <cstddef>
#include <cstdint>
#include <fstream>
#include <iostream>
#include <string>
#include <utility>
#include <vector>

#include <tudocomp/util.h>

#include <tudocomp/io/IOUtil.hpp>
#include <tudocomp/io/ViewStream.hpp>

namespace tudocomp {
namespace io {

    struct InputView;
    struct InputStream;

    /// Represents the input of an algorithm.
    ///
    /// Can be used as either a istream or a memory buffer
    /// with the as_stream or as_view methods.
    class Input {
        struct Variant {
            virtual std::unique_ptr<Variant> virtual_copy() = 0;
            virtual InputView as_view() = 0;
            virtual InputStream as_stream() = 0;
            virtual size_t size() = 0;
        };

        struct Memory: Variant {
            const uint8_t* ptr;
            size_t m_size;

            Memory(const uint8_t* ptr, size_t size): m_size(size) {
                this->ptr = ptr;
            }

            std::unique_ptr<Variant> virtual_copy() override {
                return std::make_unique<Memory>(ptr, m_size);
            }

            inline InputView as_view() override;
            inline InputStream as_stream() override;
            inline size_t size() override;
        };
        struct File: Variant {
            std::string path;
            size_t offset;

            File(std::string path, size_t offset) {
                this->path = path;
                this->offset = offset;
            }

            std::unique_ptr<Variant> virtual_copy() override {
                return std::make_unique<File>(path, offset);
            }

            inline InputView as_view() override;
            inline InputStream as_stream() override;
            inline size_t size() override;
        };

        friend class InputStream;
        friend class InputView;

        std::unique_ptr<Variant> m_data;

    public:
        struct Path { std::string path; };

        /// An empty Input
        inline Input():
            m_data(std::make_unique<Memory>(nullptr, 0)) {}

        /// Create an copy of the Input, with internal cursors
        /// set to the current position
        inline Input(const Input& other):
            m_data(std::move(other.m_data->virtual_copy())) {}

        /// Move constructor
        inline Input(Input&& other):
            m_data(std::move(other.m_data)) {}

        /// An Input referring to the contents of a file
        Input(Input::Path&& path):
            m_data(std::make_unique<File>(std::move(path.path), 0)) {}

        /// An Input referring to the contents of an vector
        Input(const std::vector<uint8_t>& buf):
            m_data(std::make_unique<Memory>(&buf[0], buf.size())) {}

        /// An Input referring to the contents of an string
        Input(const string_ref buf):
            m_data(std::make_unique<Memory>(
                    (const uint8_t*) &buf[0], buf.size())) {}

        Input& operator=(Input&& other) {
            m_data = std::move(other.m_data);
            return *this;
        }

        /// DEPRECATED
        static Input from_path(std::string path) {
            return Input(Path { path });
        }

        /// DEPRECATED
        static Input from_memory(const std::vector<uint8_t>& buf) {
            return Input(buf);
        }

        /// DEPRECATED
        static Input from_memory(const string_ref buf) {
            return Input(buf);
        }

        inline InputView as_view();
        inline InputStream as_stream();

        inline size_t size() {
            return m_data->size();
        }
    };

    class InputViewVariant {
        struct Variant {
            virtual string_ref view() = 0;
        };

        struct Memory: Variant {
            const uint8_t* ptr;
            size_t size;

            Memory(const uint8_t* ptr, size_t size) {
                this->ptr = ptr;
                this->size = size;
            }

            string_ref view() override {
                return string_ref {
                    (char*) ptr,
                    size
                };
            }
        };
        struct File: Variant {
            std::vector<uint8_t> buffer;

            File(std::vector<uint8_t>&& buffer_):
                buffer(std::move(buffer_)) {}

            string_ref view() override {
                return string_ref {
                    (char*) buffer.data(),
                    buffer.size()
                };
            }
        };

        std::unique_ptr<Variant> m_variant;

        friend class InputView;
        friend class Input;

        InputViewVariant(const InputViewVariant* other) = delete;
        InputViewVariant() = delete;

        InputViewVariant(InputViewVariant::Memory&& mem):
            m_variant(std::make_unique<Memory>(std::move(mem))) {}
        InputViewVariant(InputViewVariant::File&& s):
            m_variant(std::make_unique<File>(std::move(s))) {}
        InputViewVariant(InputViewVariant&& s):
            m_variant(std::move(s.m_variant)) {}
    };

    class InputView: InputViewVariant, public View {
        friend class Input;

        InputView(InputViewVariant&& mem):
            InputViewVariant(std::move(mem)),
            View(m_variant->view()) {}
    public:
        /// DEPRECATED
        string_ref operator* () {
            return *this;
        }

        InputView(const InputView* other) = delete;
        InputView() = delete;
    };

    inline InputView Input::Memory::as_view() {
        Input::Memory mem2 = *this;

        // advance view into memory by its whole length
        ptr += m_size;
        m_size = 0;

        return InputView {
            InputView::Memory { mem2.ptr, mem2.m_size }
        };
    }

    inline InputView Input::File::as_view() {
        // read file into buffer starting at current offset
        auto buf = read_file_to_stl_byte_container<
            std::vector<uint8_t>>(path, offset);

        // We read the whole file, so skip it on next read.
        offset += buf.size();

        return InputView {
            InputView::File {
                std::move(buf)
            }
        };
    }

    inline InputView Input::as_view() {
        return m_data->as_view();
    }

    struct InputStream {
        class Variant {
        public:
            virtual std::istream& stream() = 0;
            virtual ~Variant() {}
        };

        class Memory: public InputStream::Variant {
            ViewStream m_stream;

            Input::Memory* m_offset_back_ref;
            size_t m_start_pos;

            bool m_is_empty = false;

            friend class InputStream;

        public:
            Memory(const Memory& other) = delete;
            Memory() = delete;

            Memory(Memory&& other):
                m_stream(std::move(other.m_stream)),
                m_offset_back_ref(other.m_offset_back_ref),
                m_start_pos(other.m_start_pos) {
                other.m_is_empty = true;
            }

            Memory(ViewStream&& stream, Input::Memory* offset_back_ref):
                m_stream(std::move(stream))
            {
                m_offset_back_ref = offset_back_ref;
                m_start_pos = m_stream.stream().tellg();
            }
            virtual ~Memory() {
                if (!m_is_empty) {
                    size_t len = size_t(m_stream.stream().tellg()) - m_start_pos;
                    m_offset_back_ref->ptr += len;
                    m_offset_back_ref->m_size -= len;
                }
            }
            std::istream& stream() override {
                return m_stream.stream();
            }
        };
        class File: public InputStream::Variant {
            std::string m_path;
            std::unique_ptr<std::ifstream> m_stream;

            Input::File* m_offset_back_ref;
            size_t m_start_pos;

            friend class InputStream;
        public:
            File(const File& other) = delete;
            File() = delete;

            File(std::string&& path,
                 Input::File* offset_back_ref,
                 size_t offset)
            {
                m_path = path;
                m_stream = std::make_unique<std::ifstream>(
                    m_path, std::ios::in | std::ios::binary);
                m_stream->seekg(offset, std::ios::beg);
                m_start_pos = m_stream->tellg();
                m_offset_back_ref = offset_back_ref;
            }

            File(File&& other) {
                m_stream = std::move(other.m_stream);
                m_offset_back_ref = other.m_offset_back_ref;
                m_path = other.m_path;
                m_start_pos = other.m_start_pos;
            }

            virtual ~File() {
                if (m_stream != nullptr) {
                    auto len = size_t(m_stream->tellg()) - m_start_pos;
                    m_offset_back_ref->offset += len;
                }
            }

            std::istream& stream() override {
                return *m_stream;
            }
        };

        std::unique_ptr<InputStream::Variant> m_data;

        std::istream& operator* () {
            return m_data->stream();
        }

        InputStream(const InputStream* other) = delete;
        InputStream() = delete;

        InputStream(InputStream::Memory&& mem):
            m_data(std::move(std::make_unique<InputStream::Memory>(std::move(mem)))) {}
        InputStream(InputStream::File&& f):
            m_data(std::move(std::make_unique<InputStream::File>(std::move(f)))) {}
    };

    inline InputStream Input::Memory::as_stream() {
        return InputStream {
            InputStream::Memory {
                ViewStream {
                    (char*)ptr,
                    m_size
                },
                this
            }
        };
    }

    inline InputStream Input::File::as_stream() {
        return InputStream {
            InputStream::File {
                std::string(path),
                this,
                offset,
            }
        };
    }

    inline InputStream Input::as_stream() {
        return m_data->as_stream();
    }

    inline size_t Input::Memory::size() {
        return m_size;
    }

    inline size_t Input::File::size() {
        return read_file_size(path) - offset;
    }

}}

#endif

<commit_msg>Removed deref step for InputStream<commit_after>#ifndef _INCLUDED_INPUT_HPP
#define _INCLUDED_INPUT_HPP

#include <cstddef>
#include <cstdint>
#include <fstream>
#include <iostream>
#include <string>
#include <utility>
#include <vector>

#include <tudocomp/util.h>

#include <tudocomp/io/IOUtil.hpp>
#include <tudocomp/io/ViewStream.hpp>

namespace tudocomp {
namespace io {

    struct InputView;
    struct InputStream;

    /// Represents the input of an algorithm.
    ///
    /// Can be used as either a istream or a memory buffer
    /// with the as_stream or as_view methods.
    class Input {
        struct Variant {
            virtual std::unique_ptr<Variant> virtual_copy() = 0;
            virtual InputView as_view() = 0;
            virtual InputStream as_stream() = 0;
            virtual size_t size() = 0;
        };

        struct Memory: Variant {
            const uint8_t* ptr;
            size_t m_size;

            Memory(const uint8_t* ptr, size_t size): m_size(size) {
                this->ptr = ptr;
            }

            std::unique_ptr<Variant> virtual_copy() override {
                return std::make_unique<Memory>(ptr, m_size);
            }

            inline InputView as_view() override;
            inline InputStream as_stream() override;
            inline size_t size() override;
        };
        struct File: Variant {
            std::string path;
            size_t offset;

            File(std::string path, size_t offset) {
                this->path = path;
                this->offset = offset;
            }

            std::unique_ptr<Variant> virtual_copy() override {
                return std::make_unique<File>(path, offset);
            }

            inline InputView as_view() override;
            inline InputStream as_stream() override;
            inline size_t size() override;
        };

        friend class InputStream;
        friend class InputStreamVariant;
        friend class InputView;

        std::unique_ptr<Variant> m_data;

    public:
        struct Path { std::string path; };

        /// An empty Input
        inline Input():
            m_data(std::make_unique<Memory>(nullptr, 0)) {}

        /// Create an copy of the Input, with internal cursors
        /// set to the current position
        inline Input(const Input& other):
            m_data(std::move(other.m_data->virtual_copy())) {}

        /// Move constructor
        inline Input(Input&& other):
            m_data(std::move(other.m_data)) {}

        /// An Input referring to the contents of a file
        Input(Input::Path&& path):
            m_data(std::make_unique<File>(std::move(path.path), 0)) {}

        /// An Input referring to the contents of an vector
        Input(const std::vector<uint8_t>& buf):
            m_data(std::make_unique<Memory>(&buf[0], buf.size())) {}

        /// An Input referring to the contents of an string
        Input(const string_ref buf):
            m_data(std::make_unique<Memory>(
                    (const uint8_t*) &buf[0], buf.size())) {}

        Input& operator=(Input&& other) {
            m_data = std::move(other.m_data);
            return *this;
        }

        /// DEPRECATED
        static Input from_path(std::string path) {
            return Input(Path { path });
        }

        /// DEPRECATED
        static Input from_memory(const std::vector<uint8_t>& buf) {
            return Input(buf);
        }

        /// DEPRECATED
        static Input from_memory(const string_ref buf) {
            return Input(buf);
        }

        inline InputView as_view();
        inline InputStream as_stream();

        inline size_t size() {
            return m_data->size();
        }
    };

    class InputViewVariant {
        struct Variant {
            virtual string_ref view() = 0;
        };

        struct Memory: Variant {
            const uint8_t* ptr;
            size_t size;

            Memory(const uint8_t* ptr, size_t size) {
                this->ptr = ptr;
                this->size = size;
            }

            string_ref view() override {
                return string_ref {
                    (char*) ptr,
                    size
                };
            }
        };
        struct File: Variant {
            std::vector<uint8_t> buffer;

            File(std::vector<uint8_t>&& buffer_):
                buffer(std::move(buffer_)) {}

            string_ref view() override {
                return string_ref {
                    (char*) buffer.data(),
                    buffer.size()
                };
            }
        };

        std::unique_ptr<Variant> m_variant;

        friend class InputView;
        friend class Input;

        InputViewVariant(const InputViewVariant* other) = delete;
        InputViewVariant() = delete;

        InputViewVariant(InputViewVariant::Memory&& mem):
            m_variant(std::make_unique<Memory>(std::move(mem))) {}
        InputViewVariant(InputViewVariant::File&& s):
            m_variant(std::make_unique<File>(std::move(s))) {}
        InputViewVariant(InputViewVariant&& s):
            m_variant(std::move(s.m_variant)) {}
    };

    class InputView: InputViewVariant, public View {
        friend class Input;

        InputView(InputViewVariant&& mem):
            InputViewVariant(std::move(mem)),
            View(m_variant->view()) {}
    public:
        InputView(InputView&& mem):
            InputViewVariant(std::move(mem)),
            View(std::move(mem)) {}

        /// DEPRECATED
        string_ref operator* () {
            return *this;
        }

        InputView(const InputView* other) = delete;
        InputView() = delete;
    };

    inline InputView Input::Memory::as_view() {
        Input::Memory mem2 = *this;

        // advance view into memory by its whole length
        ptr += m_size;
        m_size = 0;

        return InputView {
            InputView::Memory { mem2.ptr, mem2.m_size }
        };
    }

    inline InputView Input::File::as_view() {
        // read file into buffer starting at current offset
        auto buf = read_file_to_stl_byte_container<
            std::vector<uint8_t>>(path, offset);

        // We read the whole file, so skip it on next read.
        offset += buf.size();

        return InputView {
            InputView::File {
                std::move(buf)
            }
        };
    }

    inline InputView Input::as_view() {
        return m_data->as_view();
    }

    class InputStreamVariant {
        class Variant {
        public:
            virtual std::istream& stream() = 0;
            virtual ~Variant() {}
        };

        class Memory: public InputStreamVariant::Variant {
            ViewStream m_stream;

            Input::Memory* m_offset_back_ref;
            size_t m_start_pos;

            bool m_is_empty = false;

            friend class InputStreamVariant;

        public:
            Memory(const Memory& other) = delete;
            Memory() = delete;

            Memory(Memory&& other):
                m_stream(std::move(other.m_stream)),
                m_offset_back_ref(other.m_offset_back_ref),
                m_start_pos(other.m_start_pos) {
                other.m_is_empty = true;
            }

            Memory(ViewStream&& stream, Input::Memory* offset_back_ref):
                m_stream(std::move(stream))
            {
                m_offset_back_ref = offset_back_ref;
                m_start_pos = m_stream.stream().tellg();
            }
            virtual ~Memory() {
                if (!m_is_empty) {
                    size_t len = size_t(m_stream.stream().tellg()) - m_start_pos;
                    m_offset_back_ref->ptr += len;
                    m_offset_back_ref->m_size -= len;
                }
            }
            std::istream& stream() override {
                return m_stream.stream();
            }
        };
        class File: public InputStreamVariant::Variant {
            std::string m_path;
            std::unique_ptr<std::ifstream> m_stream;

            Input::File* m_offset_back_ref;
            size_t m_start_pos;

            friend class InputStreamVariant;
        public:
            File(const File& other) = delete;
            File() = delete;

            File(std::string&& path,
                 Input::File* offset_back_ref,
                 size_t offset)
            {
                m_path = path;
                m_stream = std::make_unique<std::ifstream>(
                    m_path, std::ios::in | std::ios::binary);
                m_stream->seekg(offset, std::ios::beg);
                m_start_pos = m_stream->tellg();
                m_offset_back_ref = offset_back_ref;
            }

            File(File&& other) {
                m_stream = std::move(other.m_stream);
                m_offset_back_ref = other.m_offset_back_ref;
                m_path = other.m_path;
                m_start_pos = other.m_start_pos;
            }

            virtual ~File() {
                if (m_stream != nullptr) {
                    auto len = size_t(m_stream->tellg()) - m_start_pos;
                    m_offset_back_ref->offset += len;
                }
            }

            std::istream& stream() override {
                return *m_stream;
            }
        };

        std::unique_ptr<InputStreamVariant::Variant> m_variant;

        friend class InputStream;
        friend class Input;

        InputStreamVariant(const InputStreamVariant* other) = delete;
        InputStreamVariant() = delete;

        InputStreamVariant(InputStreamVariant::Memory&& mem):
            m_variant(std::move(std::make_unique<InputStreamVariant::Memory>(std::move(mem)))) {}
        InputStreamVariant(InputStreamVariant::File&& f):
            m_variant(std::move(std::make_unique<InputStreamVariant::File>(std::move(f)))) {}
        InputStreamVariant(InputStreamVariant&& s):
            m_variant(std::move(s.m_variant)) {}

    };

    class InputStream: InputStreamVariant, public std::istream {
        friend class Input;

        InputStream(InputStreamVariant&& mem):
            InputStreamVariant(std::move(mem)),
            std::istream(m_variant->stream().rdbuf()) {}
    public:
        InputStream(InputStream&& mem):
            InputStreamVariant(std::move(mem)),
            std::istream(mem.rdbuf()) {}

        /// DEPRECATED
        std::istream& operator* () {
            return *this;
        }

        InputStream(const InputStream* other) = delete;
        InputStream() = delete;
    };

    inline InputStream Input::Memory::as_stream() {
        return InputStream {
            InputStream::Memory {
                ViewStream {
                    (char*)ptr,
                    m_size
                },
                this
            }
        };
    }

    inline InputStream Input::File::as_stream() {
        return InputStream {
            InputStream::File {
                std::string(path),
                this,
                offset,
            }
        };
    }

    inline InputStream Input::as_stream() {
        return m_data->as_stream();
    }

    inline size_t Input::Memory::size() {
        return m_size;
    }

    inline size_t Input::File::size() {
        return read_file_size(path) - offset;
    }

}}

#endif

<|endoftext|>


<commit_before>//============================================================================
// vSMC/include/vsmc/core/monitor.hpp
//----------------------------------------------------------------------------
//                         vSMC: Scalable Monte Carlo
//----------------------------------------------------------------------------
// Copyright (c) 2013-2015, Yan Zhou
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//   Redistributions of source code must retain the above copyright notice,
//   this list of conditions and the following disclaimer.
//
//   Redistributions in binary form must reproduce the above copyright notice,
//   this list of conditions and the following disclaimer in the documentation
//   and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS AS IS
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
//============================================================================

#ifndef VSMC_CORE_MONITOR_HPP
#define VSMC_CORE_MONITOR_HPP

#include <vsmc/internal/common.hpp>
#include <vsmc/integrate/is_integrate.hpp>
#include <vsmc/utility/aligned_memory.hpp>

#define VSMC_RUNTIME_ASSERT_CORE_MONITOR_ID(func) \
    VSMC_RUNTIME_ASSERT((id < dim()),                                        \
            ("**Monitor::"#func"** INVALID ID NUMBER ARGUMENT"))

#define VSMC_RUNTIME_ASSERT_CORE_MONITOR_ITER(func) \
    VSMC_RUNTIME_ASSERT((iter < iter_size()),                                \
            ("**Monitor::"#func"** INVALID ITERATION NUMBER ARGUMENT"))

#define VSMC_RUNTIME_ASSERT_CORE_MONITOR_FUNCTOR(func, caller, name) \
    VSMC_RUNTIME_ASSERT(static_cast<bool>(func),                             \
            ("**Monitor::"#caller"** INVALID "#name" OBJECT"))               \

namespace vsmc {

/// \brief Monitor for Monte Carlo integration
/// \ingroup Core
template <typename T>
class Monitor
{
    public :

    typedef T value_type;
    typedef cxx11::function<
        void (std::size_t, std::size_t, const Particle<T> &, double *)>
        eval_type;

    /// \brief Construct a Monitor with an evaluation object
    ///
    /// \param dim The dimension of the Monitor, i.e., the number of variables
    /// \param eval The evaluation object of type Monitor::eval_type
    /// \param record_only The Monitor only records results instead of
    /// calculating them itself
    /// \param stage The stage of the Monitor. A Monitor may be evaluated after
    /// all steps that move the particles but before any resampling
    /// (`MonitorMove`), or the possible resampling step but before any MCMC
    /// steps (`MonitorResample`), all after all MCMC steps (`MonitorMCMC`). If
    /// a Monitor is present during initialization, then the initialization are
    /// taken as the step that moves particles, and both `MonitorResample` and
    /// `MonitorMCMC` are considered after the possible resampling.
    ///
    /// The evaluation object has the signature
    /// ~~~{.cpp}
    /// void eval (std::size_t iter, std::size_t dim, const Particle<T> &particle, double *result)
    /// ~~~
    /// where the first three arguments are passed in by the Sampler at the
    /// end of each iteration. The evaluation occurs after the possible MCMC
    /// moves. The output parameter `result` shall contain the results of the
    /// evaluation.
    ///
    /// If `record_only` is true, then the monitor only records the values
    /// stored in `result`. Otherwise, the behavior is explained below
    ///
    /// The array `result` is of length `particle.size() * dim`, and it
    /// represents a row major matrix of dimension `particle.size()` by `dim`,
    /// say \f$R\f$. Let \f$W\f$ be the vector of the normalized weights. The
    /// Monitor will be respoinsible to compute the importance sampling
    /// estimate \f$r = R^TW\f$ and record it. For example, say the purpose of
    /// the Monitor is to record the importance sampling estimates of
    /// \f$E[h(X)]\f$ where \f$h(X) = (h_1(X),\dots,h_d(X))\f$. Then `result`
    /// shall contain the evaluation of \f$h(X_i)\f$ for each \f$i\f$ from `0`
    /// to `particle.size() - 1` in the order
    /// \f$(h_1(X_0), \dots, h_d(X_0), h_1(X_1), \dots, h_d(X_1), \dots)\f$.
    ///
    /// After each evaluation, the iteration number `iter` and the imporatance
    /// sampling estimates are recorded and can be retrived by `index()` and
    /// `record()`.
    explicit Monitor (std::size_t dim, const eval_type &eval,
            bool record_only = false, MonitorStage stage = MonitorMCMC) :
        dim_(dim), eval_(eval), recording_(true),
        record_only_(record_only), stage_(stage), name_(dim) {}

    /// \brief The dimension of the Monitor
    std::size_t dim () const {return dim_;}

    /// \brief If this is a record only Monitor
    bool record_only () const {return record_only_;}

    /// \brief The stage of the Montior
    MonitorStage stage () const {return stage_;}

    /// \brief The number of iterations has been recorded
    ///
    /// \details
    /// This is not necessarily the same as Sampler<T>::iter_size. For
    /// example, a Monitor can be added only after a certain time point of the
    /// sampler's iterations. Also the Monitor can be turned off for a period
    /// during the iterations.
    std::size_t iter_size () const {return index_.size();}

    /// \brief Reserve space for a specified number of iterations
    void reserve (std::size_t num)
    {
        index_.reserve(num);
        record_.reserve(dim_ * num);
    }

    /// \brief Whether the evaluation object is valid
    bool empty () const {return !static_cast<bool>(eval_);}

    /// \brief Read and write access to the names of variables
    ///
    /// \details
    /// By default, each variable of a Monitor is unnamed and the returned
    /// string is empty. However, the user can selectively set the names of
    /// each variable. This effect how Sampler will print the headers of the
    /// summary table.
    std::string &name (std::size_t id)
    {
        VSMC_RUNTIME_ASSERT_CORE_MONITOR_ID(name);

        return name_[id];
    }

    /// \brief Read only access to the names of variables
    const std::string &name (std::size_t id) const
    {
        VSMC_RUNTIME_ASSERT_CORE_MONITOR_ID(name);

        return name_[id];
    }

    /// \brief Get the iteration index of the sampler of a given Monitor
    /// iteration
    ///
    /// \details
    /// For example, if a Monitor is only added to the sampler at the
    /// sampler's iteration `siter`. Then `index(0)` will be `siter` and so
    /// on. If the Monitor is added before the sampler's initialization and
    /// continued to be evaluated during the iterations without calling
    /// `turnoff()`, then iter(iter) shall just be `iter`.
    std::size_t index (std::size_t iter) const
    {
        VSMC_RUNTIME_ASSERT_CORE_MONITOR_ITER(index);

        return index_[iter];
    }

    /// \brief Get the latest Monte Carlo integration record of a given
    /// variable
    ///
    /// \details
    /// For a `dim` dimension Monitor, `id` shall be 0 to `dim` - 1
    double record (std::size_t id) const
    {
        std::size_t iter = iter_size() ? iter_size() - 1 : iter_size();
        VSMC_RUNTIME_ASSERT_CORE_MONITOR_ID(record);
        VSMC_RUNTIME_ASSERT_CORE_MONITOR_ITER(record);

        return record_[iter * dim_ + id];
    }

    /// \brief Get the Monte Carlo integration record of a given variable and
    /// the Monitor iteration
    ///
    /// \details
    /// For a `dim` dimension Monitor, `id` shall be 0 to `dim` - 1
    double record (std::size_t id, std::size_t iter) const
    {
        VSMC_RUNTIME_ASSERT_CORE_MONITOR_ID(record);
        VSMC_RUNTIME_ASSERT_CORE_MONITOR_ITER(record);

        return record_[iter * dim_ + id];
    }

    /// \brief Read the index history through an output iterator
    template <typename OutputIter>
    void read_index (OutputIter first) const
    {std::copy(index_.begin(), index_.end(), first);}

    /// \brief Read only access to the raw data of the index vector
    const std::size_t *index_data () const {return &index_[0];}

    /// \brief Read only access to the raw data of records (a row major matrix)
    const double *record_data () const {return &record_[0];}

    /// \brief Read the record history for a given variable through an output
    /// iterator
    template <typename OutputIter>
    void read_record (std::size_t id, OutputIter first) const
    {
        const std::size_t N = iter_size();
        const double *riter = &record_[id];
        for (std::size_t i = 0; i != N; ++i, ++first, riter += dim_)
            *first = *riter;
    }

    /// \brief Read the record history of all variables through an array of
    /// output iterators
    ///
    /// \param first An iterator of container of output iterators
    template <typename OutputIterIter>
    void read_record_matrix (OutputIterIter first) const
    {
        for (std::size_t d = 0; d != dim_; ++d, ++first)
            read_record(d, *first);
    }

    /// \brief Read the record history of all variables through an output
    /// iterator
    ///
    /// \param first The output iterator
    ///
    /// For example, say `first` is of type `double *`, then if `order ==
    /// ColMajor`, then, `first[j * iter_size() + i] == record(i, j)`.
    /// Otherwise, if `order == RowMajor`, then `first[i * dim() + j] ==
    /// record(i, j)`. That is, the output is an `iter_size()` by `dim()`
    /// matrix, with the usual meaning of column or row major order.
    template <MatrixOrder Order, typename OutputIter>
    void read_record_matrix (OutputIter first) const
    {
        const std::size_t N = iter_size();
        if (Order == ColMajor) {
            for (std::size_t d = 0; d != dim_; ++d) {
                const double *riter = &record_[d];
                for (std::size_t i = 0; i != N; ++i, ++first, riter += dim_)
                    *first = *riter;
            }
        }

        if (Order == RowMajor)
            std::copy(record_.begin(), record_.end(), first);
    }

    /// \brief Set a new evaluation object of type eval_type
    void set_eval (const eval_type &new_eval) {eval_ = new_eval;}

    /// \brief Perform the evaluation for a given iteration and a Particle<T>
    /// object.
    ///
    /// \details
    /// This function is called by a Sampler at the end of each
    /// iteration. It does nothing if `recording()` returns `false`. Otherwise
    /// it use the user defined evaluation object to compute results. When a
    /// Monitor is constructed, `recording()` always returns `true`. It can be
    /// turned off by `turn_off()` and turned on later by `turn_on()`.
    void eval (std::size_t iter, const Particle<T> &particle,
            MonitorStage stage)
    {
        if (!recording_)
            return;

        if (stage != stage_)
            return;

        VSMC_RUNTIME_ASSERT_CORE_MONITOR_FUNCTOR(eval_, eval, EVALUATION);

        result_.resize(dim_);
        double *const rptr = &result_[0];
        if (record_only_) {
            eval_(iter, dim_, particle, rptr);
            push_back(iter);

            return;
        }

        const std::size_t N = static_cast<std::size_t>(particle.size());
        buffer_.resize(N * dim_);
        double *const bptr = &buffer_[0];
        const double *const wptr = particle.weight_set().weight_data();
        eval_(iter, dim_, particle, bptr);
        is_integrate_(static_cast<ISIntegrate::size_type>(N),
                static_cast<ISIntegrate::size_type>(dim_), bptr, wptr, rptr);
        push_back(iter);
    }

    /// \brief Clear all records of the index and integrations
    void clear ()
    {
        index_.clear();
        record_.clear();
    }

    /// \brief Whether the Monitor is actively recording results
    bool recording () const {return recording_;}

    /// \brief Turn on the recording
    void turn_on () {recording_ = true;}

    /// \brief Turn off the recording
    void turn_off () {recording_ = false;}

    private :

    std::size_t dim_;
    eval_type eval_;
    bool recording_;
    bool record_only_;
    MonitorStage stage_;
    std::vector<std::string> name_;
    std::vector<std::size_t> index_;
    std::vector<double, AlignedAllocator<double> > record_;
    std::vector<double, AlignedAllocator<double> > result_;
    std::vector<double, AlignedAllocator<double> > buffer_;
    ISIntegrate is_integrate_;

    void push_back (std::size_t iter)
    {
        index_.push_back(iter);
        record_.insert(result_.end(), result_.begin(), result_.end());
    }
}; // class Monitor

} // namespace vsmc

#endif // VSMC_CORE_MONITOR_HPP
<commit_msg>Fix record_ insertion<commit_after>//============================================================================
// vSMC/include/vsmc/core/monitor.hpp
//----------------------------------------------------------------------------
//                         vSMC: Scalable Monte Carlo
//----------------------------------------------------------------------------
// Copyright (c) 2013-2015, Yan Zhou
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//   Redistributions of source code must retain the above copyright notice,
//   this list of conditions and the following disclaimer.
//
//   Redistributions in binary form must reproduce the above copyright notice,
//   this list of conditions and the following disclaimer in the documentation
//   and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS AS IS
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
//============================================================================

#ifndef VSMC_CORE_MONITOR_HPP
#define VSMC_CORE_MONITOR_HPP

#include <vsmc/internal/common.hpp>
#include <vsmc/integrate/is_integrate.hpp>
#include <vsmc/utility/aligned_memory.hpp>

#define VSMC_RUNTIME_ASSERT_CORE_MONITOR_ID(func) \
    VSMC_RUNTIME_ASSERT((id < dim()),                                        \
            ("**Monitor::"#func"** INVALID ID NUMBER ARGUMENT"))

#define VSMC_RUNTIME_ASSERT_CORE_MONITOR_ITER(func) \
    VSMC_RUNTIME_ASSERT((iter < iter_size()),                                \
            ("**Monitor::"#func"** INVALID ITERATION NUMBER ARGUMENT"))

#define VSMC_RUNTIME_ASSERT_CORE_MONITOR_FUNCTOR(func, caller, name) \
    VSMC_RUNTIME_ASSERT(static_cast<bool>(func),                             \
            ("**Monitor::"#caller"** INVALID "#name" OBJECT"))               \

namespace vsmc {

/// \brief Monitor for Monte Carlo integration
/// \ingroup Core
template <typename T>
class Monitor
{
    public :

    typedef T value_type;
    typedef cxx11::function<
        void (std::size_t, std::size_t, const Particle<T> &, double *)>
        eval_type;

    /// \brief Construct a Monitor with an evaluation object
    ///
    /// \param dim The dimension of the Monitor, i.e., the number of variables
    /// \param eval The evaluation object of type Monitor::eval_type
    /// \param record_only The Monitor only records results instead of
    /// calculating them itself
    /// \param stage The stage of the Monitor. A Monitor may be evaluated after
    /// all steps that move the particles but before any resampling
    /// (`MonitorMove`), or the possible resampling step but before any MCMC
    /// steps (`MonitorResample`), all after all MCMC steps (`MonitorMCMC`). If
    /// a Monitor is present during initialization, then the initialization are
    /// taken as the step that moves particles, and both `MonitorResample` and
    /// `MonitorMCMC` are considered after the possible resampling.
    ///
    /// The evaluation object has the signature
    /// ~~~{.cpp}
    /// void eval (std::size_t iter, std::size_t dim, const Particle<T> &particle, double *result)
    /// ~~~
    /// where the first three arguments are passed in by the Sampler at the
    /// end of each iteration. The evaluation occurs after the possible MCMC
    /// moves. The output parameter `result` shall contain the results of the
    /// evaluation.
    ///
    /// If `record_only` is true, then the monitor only records the values
    /// stored in `result`. Otherwise, the behavior is explained below
    ///
    /// The array `result` is of length `particle.size() * dim`, and it
    /// represents a row major matrix of dimension `particle.size()` by `dim`,
    /// say \f$R\f$. Let \f$W\f$ be the vector of the normalized weights. The
    /// Monitor will be respoinsible to compute the importance sampling
    /// estimate \f$r = R^TW\f$ and record it. For example, say the purpose of
    /// the Monitor is to record the importance sampling estimates of
    /// \f$E[h(X)]\f$ where \f$h(X) = (h_1(X),\dots,h_d(X))\f$. Then `result`
    /// shall contain the evaluation of \f$h(X_i)\f$ for each \f$i\f$ from `0`
    /// to `particle.size() - 1` in the order
    /// \f$(h_1(X_0), \dots, h_d(X_0), h_1(X_1), \dots, h_d(X_1), \dots)\f$.
    ///
    /// After each evaluation, the iteration number `iter` and the imporatance
    /// sampling estimates are recorded and can be retrived by `index()` and
    /// `record()`.
    explicit Monitor (std::size_t dim, const eval_type &eval,
            bool record_only = false, MonitorStage stage = MonitorMCMC) :
        dim_(dim), eval_(eval), recording_(true),
        record_only_(record_only), stage_(stage), name_(dim) {}

    /// \brief The dimension of the Monitor
    std::size_t dim () const {return dim_;}

    /// \brief If this is a record only Monitor
    bool record_only () const {return record_only_;}

    /// \brief The stage of the Montior
    MonitorStage stage () const {return stage_;}

    /// \brief The number of iterations has been recorded
    ///
    /// \details
    /// This is not necessarily the same as Sampler<T>::iter_size. For
    /// example, a Monitor can be added only after a certain time point of the
    /// sampler's iterations. Also the Monitor can be turned off for a period
    /// during the iterations.
    std::size_t iter_size () const {return index_.size();}

    /// \brief Reserve space for a specified number of iterations
    void reserve (std::size_t num)
    {
        index_.reserve(num);
        record_.reserve(dim_ * num);
    }

    /// \brief Whether the evaluation object is valid
    bool empty () const {return !static_cast<bool>(eval_);}

    /// \brief Read and write access to the names of variables
    ///
    /// \details
    /// By default, each variable of a Monitor is unnamed and the returned
    /// string is empty. However, the user can selectively set the names of
    /// each variable. This effect how Sampler will print the headers of the
    /// summary table.
    std::string &name (std::size_t id)
    {
        VSMC_RUNTIME_ASSERT_CORE_MONITOR_ID(name);

        return name_[id];
    }

    /// \brief Read only access to the names of variables
    const std::string &name (std::size_t id) const
    {
        VSMC_RUNTIME_ASSERT_CORE_MONITOR_ID(name);

        return name_[id];
    }

    /// \brief Get the iteration index of the sampler of a given Monitor
    /// iteration
    ///
    /// \details
    /// For example, if a Monitor is only added to the sampler at the
    /// sampler's iteration `siter`. Then `index(0)` will be `siter` and so
    /// on. If the Monitor is added before the sampler's initialization and
    /// continued to be evaluated during the iterations without calling
    /// `turnoff()`, then iter(iter) shall just be `iter`.
    std::size_t index (std::size_t iter) const
    {
        VSMC_RUNTIME_ASSERT_CORE_MONITOR_ITER(index);

        return index_[iter];
    }

    /// \brief Get the latest Monte Carlo integration record of a given
    /// variable
    ///
    /// \details
    /// For a `dim` dimension Monitor, `id` shall be 0 to `dim` - 1
    double record (std::size_t id) const
    {
        std::size_t iter = iter_size() ? iter_size() - 1 : iter_size();
        VSMC_RUNTIME_ASSERT_CORE_MONITOR_ID(record);
        VSMC_RUNTIME_ASSERT_CORE_MONITOR_ITER(record);

        return record_[iter * dim_ + id];
    }

    /// \brief Get the Monte Carlo integration record of a given variable and
    /// the Monitor iteration
    ///
    /// \details
    /// For a `dim` dimension Monitor, `id` shall be 0 to `dim` - 1
    double record (std::size_t id, std::size_t iter) const
    {
        VSMC_RUNTIME_ASSERT_CORE_MONITOR_ID(record);
        VSMC_RUNTIME_ASSERT_CORE_MONITOR_ITER(record);

        return record_[iter * dim_ + id];
    }

    /// \brief Read the index history through an output iterator
    template <typename OutputIter>
    void read_index (OutputIter first) const
    {std::copy(index_.begin(), index_.end(), first);}

    /// \brief Read only access to the raw data of the index vector
    const std::size_t *index_data () const {return &index_[0];}

    /// \brief Read only access to the raw data of records (a row major matrix)
    const double *record_data () const {return &record_[0];}

    /// \brief Read the record history for a given variable through an output
    /// iterator
    template <typename OutputIter>
    void read_record (std::size_t id, OutputIter first) const
    {
        const std::size_t N = iter_size();
        const double *riter = &record_[id];
        for (std::size_t i = 0; i != N; ++i, ++first, riter += dim_)
            *first = *riter;
    }

    /// \brief Read the record history of all variables through an array of
    /// output iterators
    ///
    /// \param first An iterator of container of output iterators
    template <typename OutputIterIter>
    void read_record_matrix (OutputIterIter first) const
    {
        for (std::size_t d = 0; d != dim_; ++d, ++first)
            read_record(d, *first);
    }

    /// \brief Read the record history of all variables through an output
    /// iterator
    ///
    /// \param first The output iterator
    ///
    /// For example, say `first` is of type `double *`, then if `order ==
    /// ColMajor`, then, `first[j * iter_size() + i] == record(i, j)`.
    /// Otherwise, if `order == RowMajor`, then `first[i * dim() + j] ==
    /// record(i, j)`. That is, the output is an `iter_size()` by `dim()`
    /// matrix, with the usual meaning of column or row major order.
    template <MatrixOrder Order, typename OutputIter>
    void read_record_matrix (OutputIter first) const
    {
        const std::size_t N = iter_size();
        if (Order == ColMajor) {
            for (std::size_t d = 0; d != dim_; ++d) {
                const double *riter = &record_[d];
                for (std::size_t i = 0; i != N; ++i, ++first, riter += dim_)
                    *first = *riter;
            }
        }

        if (Order == RowMajor)
            std::copy(record_.begin(), record_.end(), first);
    }

    /// \brief Set a new evaluation object of type eval_type
    void set_eval (const eval_type &new_eval) {eval_ = new_eval;}

    /// \brief Perform the evaluation for a given iteration and a Particle<T>
    /// object.
    ///
    /// \details
    /// This function is called by a Sampler at the end of each
    /// iteration. It does nothing if `recording()` returns `false`. Otherwise
    /// it use the user defined evaluation object to compute results. When a
    /// Monitor is constructed, `recording()` always returns `true`. It can be
    /// turned off by `turn_off()` and turned on later by `turn_on()`.
    void eval (std::size_t iter, const Particle<T> &particle,
            MonitorStage stage)
    {
        if (!recording_)
            return;

        if (stage != stage_)
            return;

        VSMC_RUNTIME_ASSERT_CORE_MONITOR_FUNCTOR(eval_, eval, EVALUATION);

        result_.resize(dim_);
        double *const rptr = &result_[0];
        if (record_only_) {
            eval_(iter, dim_, particle, rptr);
            push_back(iter);

            return;
        }

        const std::size_t N = static_cast<std::size_t>(particle.size());
        buffer_.resize(N * dim_);
        double *const bptr = &buffer_[0];
        const double *const wptr = particle.weight_set().weight_data();
        eval_(iter, dim_, particle, bptr);
        is_integrate_(static_cast<ISIntegrate::size_type>(N),
                static_cast<ISIntegrate::size_type>(dim_), bptr, wptr, rptr);
        push_back(iter);
    }

    /// \brief Clear all records of the index and integrations
    void clear ()
    {
        index_.clear();
        record_.clear();
    }

    /// \brief Whether the Monitor is actively recording results
    bool recording () const {return recording_;}

    /// \brief Turn on the recording
    void turn_on () {recording_ = true;}

    /// \brief Turn off the recording
    void turn_off () {recording_ = false;}

    private :

    std::size_t dim_;
    eval_type eval_;
    bool recording_;
    bool record_only_;
    MonitorStage stage_;
    std::vector<std::string> name_;
    std::vector<std::size_t> index_;
    std::vector<double, AlignedAllocator<double> > record_;
    std::vector<double, AlignedAllocator<double> > result_;
    std::vector<double, AlignedAllocator<double> > buffer_;
    ISIntegrate is_integrate_;

    void push_back (std::size_t iter)
    {
        index_.push_back(iter);
        record_.insert(record_.end(), result_.begin(), result_.end());
    }
}; // class Monitor

} // namespace vsmc

#endif // VSMC_CORE_MONITOR_HPP
<|endoftext|>


<commit_before>
#include <iostream>
#include <fstream>
#include <utility>
#include <ctime>

// TCLAP
#include "tclap/CmdLine.h"

#include "cereal/archives/json.hpp"

#include "rb-filesystem.hpp"

#include <sdsl/bit_vectors.hpp>
#include <cstdio>

#include <cstdlib>

#include <libgen.h>
#include <sparsepp/spp.h>
#include <boost/dynamic_bitset.hpp>
using spp::sparse_hash_map;
// Custom Headers
//#include "uint128_t.hpp"
//#include "debug.h"
#include "kmer.hpp"

//using namespace std;
//using namespace sdsl;

#include <cstdlib>
#include <sys/timeb.h>
#include "rb-pack-color.hpp"
#include <bitset>
#include "rb-vec.hpp"
int getMilliCount()
{
    timeb tb;
    ftime(&tb);
    int nCount = tb.millitm + (tb.time & 0xfffff) * 1000;
    return nCount;
}

int getMilliSpan(int nTimeStart)
{
    int nSpan = getMilliCount() - nTimeStart;
    if(nSpan < 0)
        nSpan += 0x100000 * 1000;
    return nSpan;
}

std::string file_extension = ".<extension>";

void parse_arguments(int argc, char **argv, parameters_t & params)
{
    TCLAP::CmdLine cmd("Cosmo Copyright (c) Alex Bowe (alexbowe.com) 2014", ' ', VERSION);
    TCLAP::UnlabeledValueArg<std::string> input_filename_arg("input",
                                                             "Input file. Currently only supports DSK's binary format (for k<=64).", true, "", "input_file", cmd);
    TCLAP::UnlabeledValueArg<std::string> num_colors_arg("num_colors",
                                                         "Number of colors", true, "", "num colors", cmd);
    TCLAP::UnlabeledValueArg<std::string> res_dir_arg("dir",
                                                             "Result directory. Should have created the directory first.", true, "", "res_dir", cmd);
	TCLAP::UnlabeledValueArg<std::string> pass_arg("pass", "1pass or 2pass", false, "", "pass", cmd);
    cmd.parse( argc, argv );
    params.input_filename  = input_filename_arg.getValue();
    params.num_colors  = atoi(num_colors_arg.getValue().c_str());
	params.res_dir = res_dir_arg.getValue();
	params.pass = pass_arg.getValue();
}

void deserialize_color_bv(std::ifstream &colorfile, color_bv &value)
{
    colorfile.read((char *)&value, sizeof(color_bv));
}

bool serialize_info(uint64_t num_colors,
		    uint64_t num_edges,
			uint64_t num_eqCls,
		    std::string label_type,
		    std::string select_type,
		    std::string eqtable_type,
		    std::string res_dir) {

	std::string jsonFileName = res_dir + "/info.json";
	std::ofstream jsonFile(jsonFileName);
	{
	cereal::JSONOutputArchive archive(jsonFile);
	archive(cereal::make_nvp("label_type", label_type));
	archive(cereal::make_nvp("select_type", select_type));
	archive(cereal::make_nvp("eqtable_type", eqtable_type));
	archive(cereal::make_nvp("num_colors", num_colors));
	archive(cereal::make_nvp("num_edges", num_edges));
	archive(cereal::make_nvp("num_eqCls", num_eqCls));
	}
	jsonFile.close();
	return true;
}

template <class T1, class T2, class T3>
class ColorPacker {
	public:
			T1 lblvec;
			T2 rnkvec;
			T3 eqTvec;
	public:
			ColorPacker(uint64_t eqBitSize, uint64_t lblBitSize) :
				lblvec(lblBitSize), rnkvec(lblBitSize+1), eqTvec(eqBitSize) {
				//TODO fillout info file
				// 	color cnt
				// 	kmer cnt
				// 	...
			}

			size_t insertColorLabel(uint64_t num, uint64_t pos) {
				// most significant bit of number goes down to the end of the bitset
				uint8_t nbits = static_cast<uint8_t>(floor(log2(num+2)));
			    uint64_t lbl = num - ((1<<nbits) - 2);
				lblvec.setInt(pos, lbl, nbits);
				return pos + nbits;
				/*uint8_t nbits = static_cast<uint8_t>(num==0?1:ceil(log2(num+1)));
				lblvec.setInt(pos, num, nbits);
				return pos + nbits;
				*/
			}

			bool storeAll(std::string dir, uint64_t bitvecSize, uint64_t eqClsSize) {
				return eqTvec.serialize(dir + "/eqTable", eqClsSize) && lblvec.serialize(dir + "/lbl", bitvecSize) && rnkvec.serialize(dir + "/rnk", bitvecSize+1);
			}
};

int main(int argc, char * argv[])
{
	bool sort = true;
	bool compress = true;
    std::cerr << "pack-color compiled with supported colors=" << NUM_COLS << std::endl;
    std::cerr <<"Starting" << std::endl;
    parameters_t params;
    parse_arguments(argc, argv, params);
	if (params.pass == "1pass") sort = false; //Anything else means apply sorting!! HeHe!!
    if (!rainbowfish::fs::FileExists(params.res_dir.c_str())) {
	rainbowfish::fs::MakeDir(params.res_dir.c_str());
    }
    const char * file_name = params.input_filename.c_str();
    std::cerr << "file name: " << file_name << std::endl;

	const char * res_dir = params.res_dir.c_str();//"bitvectors";
	// Open File
    std::ifstream colorfile(file_name, std::ios::in|std::ios::binary);

    colorfile.seekg(0, colorfile.end);
    size_t end = colorfile.tellg();
    std::cerr << "file size: " << end << std::endl;
    std::cerr << "sizeof(color_bv): " << sizeof(color_bv) << std::endl;
    size_t num_color = params.num_colors;
    size_t num_edges = end / sizeof(color_bv);

	int startTime = getMilliCount();
	int checkPointTime = getMilliCount();
	int allocationTime = getMilliCount();
	//FIRST ROUND going over all edges
	// Read file and fill out equivalence classes
    std::cerr << "edges: " << num_edges << " colors: " << num_color << " Total: " << num_edges * num_color << std::endl;
    sparse_hash_map<color_bv, uint64_t> eqCls;
    std::vector<std::pair<color_bv, uint64_t>> eqClsVec;
//	ColorPacker<RBVecCompressed, RBVecCompressed, RBVecCompressed> * cp;
	ColorPacker<RBVec, RBVec, RBVec> * cp;
	uint64_t curPos = 0;
	if (sort) {
    		colorfile.seekg(0, colorfile.beg);
			for (size_t i=0; i < num_edges; i++) {
				if (i % 100000000 == 0) {
					std::cerr<<getMilliSpan(checkPointTime) << " ms : "<<i<<std::endl;
					checkPointTime = getMilliCount();
				}
				color_bv value;
				deserialize_color_bv(colorfile, value);
				if (eqCls.find(value)==eqCls.end())
					eqCls[value] = 1;
				else
					eqCls[value] = eqCls[value]+1;
			}
			std::cerr << getMilliSpan(allocationTime) << " ms : Succinct builder object allocated" << std::endl;
			checkPointTime = getMilliCount();
			// Put data in hashmap to vector for further probable sorting!!
			eqClsVec.reserve(eqCls.size());
			for (const auto& c : eqCls) { eqClsVec.push_back(c); }

			// sort the hashmap
			auto cmp = [](std::pair<color_bv, uint64_t> const & a, std::pair<color_bv, uint64_t> const & b)
			{
				return a.second > b.second;
			};
			std::sort(eqClsVec.begin(), eqClsVec.end(), cmp);

			// replacing labels instead of k-mer counts as the hash map values
			int lbl = 0;
			size_t totalBits = 0;
			uint64_t total_edges = 0;
			for (const auto& c : eqClsVec) {
				std::cout <<lbl<< ","<< c.second<<"\n ";
				total_edges += c.second;
				//for (uint64_t k=0;k<num_color;k++) if (c.first[k] == true) std::cout<<k<<" ";
				//std::cout<<"\n";
				totalBits += (lbl==0?c.second:ceil(log2(lbl+1))*c.second);
				eqCls[c.first] = lbl++;
			}
			std::cerr << getMilliSpan(checkPointTime) << " ms : (Sorting eq vector and ) assigning a label to each eq class." << std::endl;
			std::cerr << " Total edge vs total edge: "<<total_edges<<" vs "<< num_edges<<"\n";
			size_t vecBits = totalBits;
			totalBits *= 2;
			totalBits += num_color * eqCls.size();
			std::cerr << "total bits: " << totalBits << " or " << totalBits/(8*pow(1024,2)) << " MB\n";

//			cp = new ColorPacker<RBVecCompressed, RBVecCompressed, RBVecCompressed>(eqCls.size()*num_color, vecBits);
			cp = new ColorPacker<RBVec, RBVec, RBVec>(eqCls.size()*num_color, vecBits);

			// SECOND ROUND going over all edges
			checkPointTime = getMilliCount();
			int packStartTime = getMilliCount();
			// create label & rank vectors
			colorfile.seekg(0, colorfile.beg);
			for (size_t i=0; i < num_edges; i++) {
				if (i % 100000000 == 0) {
						std::cerr<<getMilliSpan(checkPointTime) << " ms : "<<i<<std::endl;
						checkPointTime = getMilliCount();
				}
				color_bv value;
				deserialize_color_bv(colorfile, value);
				(cp->rnkvec).set(curPos);
				curPos = cp->insertColorLabel(eqCls[value], curPos);
			}
			(cp->rnkvec).set(curPos);
			std::cerr << "\n" << getMilliSpan(packStartTime) << " ms : Packing label & rank into bitvector." << std::endl;
	}
	else {
		checkPointTime = getMilliCount();
		int packStartTime = getMilliCount();
		// create label & rank vectors
		colorfile.seekg(0, colorfile.beg);
//		cp = new ColorPacker<RBVecCompressed, RBVecCompressed, RBVecCompressed>(num_color*num_color, 2*num_edges*num_color);
		cp = new ColorPacker<RBVec, RBVec, RBVec>(num_color*num_color, 2*num_edges*num_color);

		for (size_t i=0; i < num_edges; i++) {
				if (i % 100000000 == 0) {
						std::cerr<<getMilliSpan(checkPointTime) << " ms : "<<i<<std::endl;
						checkPointTime = getMilliCount();
				}
				color_bv value;
				deserialize_color_bv(colorfile, value);
				if (eqCls.find(value)==eqCls.end()) {
					eqCls[value] = eqClsVec.size();
					eqClsVec.push_back(std::make_pair(value, eqCls[value]));
				}
				(cp->rnkvec).set(curPos);
				curPos = cp->insertColorLabel(eqCls[value], curPos);
			}
			(cp->rnkvec).set(curPos);
			std::cerr << "\n" << getMilliSpan(packStartTime) << " ms : Packing label & rank into bitvector." << std::endl;
	}
	// pack eqTable in bitvector
	checkPointTime = getMilliCount();
	uint64_t i = 0;
	for (const auto& c : eqClsVec) {
		for (size_t j = 0; j < num_color; ++j) {
			if (c.first[j]) (cp->eqTvec).set(i);
			i++;
		}
	}
	std::cerr << getMilliSpan(checkPointTime) << " ms : Packing eq. table into bitvector." << std::endl;

	checkPointTime = getMilliCount();
	cp->storeAll(res_dir, curPos, eqCls.size()*num_color);
	std::cerr << getMilliSpan(checkPointTime) << " ms : Storing all three bitvectors." << std::endl << std::endl;
	serialize_info(num_color, num_edges, eqCls.size(), "uncompressed", "compressed", "compressed", res_dir);
	std::cerr << getMilliSpan(startTime)/1000.0 << " s : Total Time." << std::endl;
}
<commit_msg>Use __builtin_clzll to calculate log2.<commit_after>
#include <iostream>
#include <fstream>
#include <utility>
#include <ctime>

// TCLAP
#include "tclap/CmdLine.h"

#include "cereal/archives/json.hpp"

#include "rb-filesystem.hpp"

#include <sdsl/bit_vectors.hpp>
#include <cstdio>

#include <cstdlib>

#include <libgen.h>
#include <sparsepp/spp.h>
#include <boost/dynamic_bitset.hpp>
using spp::sparse_hash_map;
// Custom Headers
//#include "uint128_t.hpp"
//#include "debug.h"
#include "kmer.hpp"

//using namespace std;
//using namespace sdsl;

#include <cstdlib>
#include <sys/timeb.h>
#include "rb-pack-color.hpp"
#include <bitset>
#include "rb-vec.hpp"

#define LOG2(X) ((unsigned) (8*sizeof (unsigned long long) - __builtin_clzll((X)) - 1))

int getMilliCount()
{
    timeb tb;
    ftime(&tb);
    int nCount = tb.millitm + (tb.time & 0xfffff) * 1000;
    return nCount;
}

int getMilliSpan(int nTimeStart)
{
    int nSpan = getMilliCount() - nTimeStart;
    if(nSpan < 0)
        nSpan += 0x100000 * 1000;
    return nSpan;
}

std::string file_extension = ".<extension>";

void parse_arguments(int argc, char **argv, parameters_t & params)
{
    TCLAP::CmdLine cmd("Cosmo Copyright (c) Alex Bowe (alexbowe.com) 2014", ' ', VERSION);
    TCLAP::UnlabeledValueArg<std::string> input_filename_arg("input",
                                                             "Input file. Currently only supports DSK's binary format (for k<=64).", true, "", "input_file", cmd);
    TCLAP::UnlabeledValueArg<std::string> num_colors_arg("num_colors",
                                                         "Number of colors", true, "", "num colors", cmd);
    TCLAP::UnlabeledValueArg<std::string> res_dir_arg("dir",
                                                             "Result directory. Should have created the directory first.", true, "", "res_dir", cmd);
	TCLAP::UnlabeledValueArg<std::string> pass_arg("pass", "1pass or 2pass", false, "", "pass", cmd);
    cmd.parse( argc, argv );
    params.input_filename  = input_filename_arg.getValue();
    params.num_colors  = atoi(num_colors_arg.getValue().c_str());
	params.res_dir = res_dir_arg.getValue();
	params.pass = pass_arg.getValue();
}

void deserialize_color_bv(std::ifstream &colorfile, color_bv &value)
{
    colorfile.read((char *)&value, sizeof(color_bv));
}

bool serialize_info(uint64_t num_colors,
		    uint64_t num_edges,
			uint64_t num_eqCls,
		    std::string label_type,
		    std::string select_type,
		    std::string eqtable_type,
		    std::string res_dir) {

	std::string jsonFileName = res_dir + "/info.json";
	std::ofstream jsonFile(jsonFileName);
	{
	cereal::JSONOutputArchive archive(jsonFile);
	archive(cereal::make_nvp("label_type", label_type));
	archive(cereal::make_nvp("select_type", select_type));
	archive(cereal::make_nvp("eqtable_type", eqtable_type));
	archive(cereal::make_nvp("num_colors", num_colors));
	archive(cereal::make_nvp("num_edges", num_edges));
	archive(cereal::make_nvp("num_eqCls", num_eqCls));
	}
	jsonFile.close();
	return true;
}

template <class T1, class T2, class T3>
class ColorPacker {
	public:
			T1 lblvec;
			T2 rnkvec;
			T3 eqTvec;
	public:
			ColorPacker(uint64_t eqBitSize, uint64_t lblBitSize) :
				lblvec(lblBitSize), rnkvec(lblBitSize+1), eqTvec(eqBitSize) {
				//TODO fillout info file
				// 	color cnt
				// 	kmer cnt
				// 	...
			}

			size_t insertColorLabel(uint64_t num, uint64_t pos) {
				// most significant bit of number goes down to the end of the bitset
				uint8_t nbits = static_cast<uint8_t>(floor(LOG2(num+2)));
			    uint64_t lbl = num - ((1<<nbits) - 2);
				lblvec.setInt(pos, lbl, nbits);
				return pos + nbits;
				/*uint8_t nbits = static_cast<uint8_t>(num==0?1:ceil(log2(num+1)));
				lblvec.setInt(pos, num, nbits);
				return pos + nbits;
				*/
			}

			bool storeAll(std::string dir, uint64_t bitvecSize, uint64_t eqClsSize) {
				return eqTvec.serialize(dir + "/eqTable", eqClsSize) && lblvec.serialize(dir + "/lbl", bitvecSize) && rnkvec.serialize(dir + "/rnk", bitvecSize+1);
			}
};

int main(int argc, char * argv[])
{
	bool sort = true;
	bool compress = true;
    std::cerr << "pack-color compiled with supported colors=" << NUM_COLS << std::endl;
    std::cerr <<"Starting" << std::endl;
    parameters_t params;
    parse_arguments(argc, argv, params);
	if (params.pass == "1pass") sort = false; //Anything else means apply sorting!! HeHe!!
    if (!rainbowfish::fs::FileExists(params.res_dir.c_str())) {
	rainbowfish::fs::MakeDir(params.res_dir.c_str());
    }
    const char * file_name = params.input_filename.c_str();
    std::cerr << "file name: " << file_name << std::endl;

	const char * res_dir = params.res_dir.c_str();//"bitvectors";
	// Open File
    std::ifstream colorfile(file_name, std::ios::in|std::ios::binary);

    colorfile.seekg(0, colorfile.end);
    size_t end = colorfile.tellg();
    std::cerr << "file size: " << end << std::endl;
    std::cerr << "sizeof(color_bv): " << sizeof(color_bv) << std::endl;
    size_t num_color = params.num_colors;
    size_t num_edges = end / sizeof(color_bv);

	int startTime = getMilliCount();
	int checkPointTime = getMilliCount();
	int allocationTime = getMilliCount();
	//FIRST ROUND going over all edges
	// Read file and fill out equivalence classes
    std::cerr << "edges: " << num_edges << " colors: " << num_color << " Total: " << num_edges * num_color << std::endl;
	ColorPacker<RBVecCompressed, RBVecCompressed, RBVecCompressed> * cp;
//	ColorPacker<RBVec, RBVec, RBVec> * cp;
    std::vector<std::pair<color_bv, uint64_t>> eqClsVec;
	uint64_t curPos = 0;
	if (sort) {
    		sparse_hash_map<color_bv, uint64_t> eqCls;
    		colorfile.seekg(0, colorfile.beg);
			for (size_t i=0; i < num_edges; i++) {
				if (i % 100000000 == 0) {
					std::cerr<<getMilliSpan(checkPointTime) << " ms : "<<i<<std::endl;
					checkPointTime = getMilliCount();
				}
				color_bv value;
				deserialize_color_bv(colorfile, value);
				if (eqCls.find(value)==eqCls.end())
					eqCls[value] = 1;
				else
					eqCls[value] = eqCls[value]+1;
			}
			std::cerr << getMilliSpan(allocationTime) << " ms : Succinct builder object allocated" << std::endl;
			checkPointTime = getMilliCount();
			// Put data in hashmap to vector for further probable sorting!!
			eqClsVec.reserve(eqCls.size());
			for (const auto& c : eqCls) { eqClsVec.push_back(c); }

			// sort the hashmap
			auto cmp = [](std::pair<color_bv, uint64_t> const & a, std::pair<color_bv, uint64_t> const & b)
			{
				return a.second > b.second;
			};
			std::sort(eqClsVec.begin(), eqClsVec.end(), cmp);

			// replacing labels instead of k-mer counts as the hash map values
			int lbl = 0;
			size_t totalBits = 0;
			uint64_t total_edges = 0;
			for (const auto& c : eqClsVec) {
				std::cout <<lbl<< " , "<< c.second<<"\n ";
				total_edges += c.second;
				//for (uint64_t k=0;k<num_color;k++) if (c.first[k] == true) std::cout<<k<<" ";
				//std::cout<<"\n";
				totalBits += (lbl==0?c.second:ceil(log2(lbl+1))*c.second);
				eqCls[c.first] = lbl++;
			}
			std::cerr << getMilliSpan(checkPointTime) << " ms : (Sorting eq vector and ) assigning a label to each eq class." << std::endl;
			std::cerr << " Total edge vs total edge: "<<total_edges<<" vs "<< num_edges<<"\n";
			size_t vecBits = totalBits;
			totalBits *= 2;
			totalBits += num_color * eqCls.size();
			std::cerr << "total bits: " << totalBits << " or " << totalBits/(8*pow(1024,2)) << " MB\n";

			cp = new ColorPacker<RBVecCompressed, RBVecCompressed, RBVecCompressed>(eqCls.size()*num_color, vecBits);
//			cp = new ColorPacker<RBVec, RBVec, RBVec>(eqCls.size()*num_color, vecBits);

			// SECOND ROUND going over all edges
			checkPointTime = getMilliCount();
			int packStartTime = getMilliCount();
			// create label & rank vectors
			colorfile.seekg(0, colorfile.beg);
			for (size_t i=0; i < num_edges; i++) {
				if (i % 100000000 == 0) {
						std::cerr<<getMilliSpan(checkPointTime) << " ms : "<<i<<std::endl;
						checkPointTime = getMilliCount();
				}
				color_bv value;
				deserialize_color_bv(colorfile, value);
				(cp->rnkvec).set(curPos);
				curPos = cp->insertColorLabel(eqCls[value], curPos);
			}
			(cp->rnkvec).set(curPos);
			std::cerr << "\n" << getMilliSpan(packStartTime) << " ms : Packing label & rank into bitvector." << std::endl;
	}
	else {
    	sparse_hash_map<color_bv, std::pair<uint64_t, uint64_t>> eqCls;
		checkPointTime = getMilliCount();
		int packStartTime = getMilliCount();
		// create label & rank vectors
		colorfile.seekg(0, colorfile.beg);
		cp = new ColorPacker<RBVecCompressed, RBVecCompressed, RBVecCompressed>(num_color*num_color, 2*num_edges*num_color);
//		cp = new ColorPacker<RBVec, RBVec, RBVec>(num_color*num_color, 2*num_edges*num_color);

		for (size_t i=0; i < num_edges; i++) {
				if (i % 100000000 == 0) {
						std::cerr<<getMilliSpan(checkPointTime) << " ms : "<<i<<std::endl;
						checkPointTime = getMilliCount();
				}
				color_bv value;
				deserialize_color_bv(colorfile, value);
				if (eqCls.find(value)==eqCls.end()) {
					eqCls[value] = std::make_pair(eqClsVec.size(), 1);
					eqClsVec.push_back(std::make_pair(value, eqCls[value].first));
				}
				else eqCls[value].second += 1;
				(cp->rnkvec).set(curPos);
				curPos = cp->insertColorLabel(eqCls[value].first, curPos);
		}
		(cp->rnkvec).set(curPos);
		std::cerr << "\n" << getMilliSpan(packStartTime) << " ms : Packing label & rank into bitvector." << std::endl;
		uint64_t eqCntr = 0;
		for (const auto& c : eqClsVec) {
			std::cout<<eqCntr++<<" , "<<(eqCls[c.first]).second<<"\n";
		}
	}
	// pack eqTable in bitvector
	checkPointTime = getMilliCount();
	uint64_t i = 0;
	for (const auto& c : eqClsVec) {
		for (size_t j = 0; j < num_color; ++j) {
			if (c.first[j]) (cp->eqTvec).set(i);
			i++;
		}
	}
	std::cerr << getMilliSpan(checkPointTime) << " ms : Packing eq. table into bitvector." << std::endl;

	checkPointTime = getMilliCount();
	cp->storeAll(res_dir, curPos, eqClsVec.size()*num_color);
	std::cerr << getMilliSpan(checkPointTime) << " ms : Storing all three bitvectors." << std::endl << std::endl;
	serialize_info(num_color, num_edges, eqClsVec.size(), "uncompressed", "compressed", "compressed", res_dir);
	std::cerr << getMilliSpan(startTime)/1000.0 << " s : Total Time." << std::endl;
}
<|endoftext|>