<commit_before>/**
 * PANDA 3D SOFTWARE
 * Copyright (c) Carnegie Mellon University.  All rights reserved.
 *
 * All use of this software is subject to the terms of the revised BSD
 * license.  You should have received a copy of this license along
 * with this source code in a file named "LICENSE."
 *
 * @file ffmpegAudioCursor.cxx
 * @author jyelon
 * @date 2007-08-01
 */

#include "config_ffmpeg.h"
#include "ffmpegAudioCursor.h"

#include "ffmpegAudio.h"
extern "C" {
  #include "libavutil/dict.h"
  #include "libavutil/opt.h"
  #include "libavcodec/avcodec.h"
  #include "libavformat/avformat.h"
}

#ifdef HAVE_SWRESAMPLE
extern "C" {
  #include "libswresample/swresample.h"
}
#endif

TypeHandle FfmpegAudioCursor::_type_handle;

#if LIBAVFORMAT_VERSION_MAJOR < 53
  #define AVMEDIA_TYPE_AUDIO CODEC_TYPE_AUDIO
#endif

#ifndef AVCODEC_MAX_AUDIO_FRAME_SIZE
// More recent versions of ffmpeg no longer define this.
#define AVCODEC_MAX_AUDIO_FRAME_SIZE 192000
#endif

/**
 * xxx
 */
FfmpegAudioCursor::
FfmpegAudioCursor(FfmpegAudio *src) :
  MovieAudioCursor(src),
  _filename(src->_filename),
  _packet(0),
  _packet_data(0),
  _format_ctx(0),
  _audio_ctx(0),
  _resample_ctx(0),
  _buffer(0),
  _buffer_alloc(0),
  _frame(0)
{
  if (!_ffvfile.open_vfs(_filename)) {
    cleanup();
    return;
  }

  _format_ctx = _ffvfile.get_format_context();
  nassertv(_format_ctx != NULL);

  if (avformat_find_stream_info(_format_ctx, NULL) < 0) {
    cleanup();
    return;
  }

  // As of libavformat version 57.41.100, AVStream.codec is deprecated in favor
  // of AVStream.codecpar.  Fortunately, the two structures have
  // similarly-named members, so we can just switch out the declaration.
#if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 41, 100)
  AVCodecParameters *codecpar;
#else
  AVCodecContext *codecpar;
#endif

  // Find the audio stream
  AVStream *stream = nullptr;
  for (int i = 0; i < (int)_format_ctx->nb_streams; i++) {
#if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 41, 100)
    codecpar = _format_ctx->streams[i]->codecpar;
#else
    codecpar = _format_ctx->streams[i]->codec;
#endif
    if (codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
      _audio_index = i;
      stream = _format_ctx->streams[i];
      break;
    }
  }

  if (stream == nullptr) {
    cleanup();
    return;
  }

  _audio_timebase = av_q2d(stream->time_base);
  _audio_rate = codecpar->sample_rate;
  _audio_channels = codecpar->channels;

  AVCodec *pAudioCodec = avcodec_find_decoder(codecpar->codec_id);
  if (pAudioCodec == 0) {
    cleanup();
    return;
  }

  _audio_ctx = avcodec_alloc_context3(pAudioCodec);

  if (_audio_ctx == nullptr) {
    cleanup();
    return;
  }

#if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 41, 100)
  avcodec_parameters_to_context(_audio_ctx, codecpar);
#else
  avcodec_copy_context(_audio_ctx, codecpar);
#endif

  AVDictionary *opts = NULL;
  av_dict_set(&opts, "request_sample_fmt", "s16", 0);
  if (avcodec_open2(_audio_ctx, pAudioCodec, NULL) < 0) {
    cleanup();
    return;
  }

  av_dict_free(&opts);

  // Set up the resample context if necessary.
  if (_audio_ctx->sample_fmt != AV_SAMPLE_FMT_S16) {
#ifdef HAVE_SWRESAMPLE
    ffmpeg_cat.debug()
      << "Codec does not use signed 16-bit sample format.  Setting up swresample context.\n";

    _resample_ctx = swr_alloc();
    av_opt_set_int(_resample_ctx, "in_channel_count", _audio_channels, 0);
    av_opt_set_int(_resample_ctx, "out_channel_count", _audio_channels, 0);
    av_opt_set_int(_resample_ctx, "in_channel_layout", _audio_ctx->channel_layout, 0);
    av_opt_set_int(_resample_ctx, "out_channel_layout", _audio_ctx->channel_layout, 0);
    av_opt_set_int(_resample_ctx, "in_sample_rate", _audio_ctx->sample_rate, 0);
    av_opt_set_int(_resample_ctx, "out_sample_rate", _audio_ctx->sample_rate, 0);
    av_opt_set_sample_fmt(_resample_ctx, "in_sample_fmt", _audio_ctx->sample_fmt, 0);
    av_opt_set_sample_fmt(_resample_ctx, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);

    if (swr_init(_resample_ctx) != 0) {
      ffmpeg_cat.error()
        << "Failed to set up resample context.\n";
      _resample_ctx = NULL;
    }
#else
    ffmpeg_cat.error()
      << "Codec does not use signed 16-bit sample format, but support for libswresample has not been enabled.\n";
#endif
  }

  _length = (_format_ctx->duration * 1.0) / AV_TIME_BASE;
  _can_seek = true;
  _can_seek_fast = true;

#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 45, 101)
  _frame = av_frame_alloc();
#else
  _frame = avcodec_alloc_frame();
#endif

#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
  _packet = av_packet_alloc();
#else
  _packet = new AVPacket;
#endif

  _buffer_size = AVCODEC_MAX_AUDIO_FRAME_SIZE / 2;
  _buffer_alloc = new int16_t[_buffer_size + 64];

  // Allocate enough space for 1024 samples per channel.
  if ((_packet == 0)||(_buffer_alloc == 0)) {
    cleanup();
    return;
  }
  memset(_packet, 0, sizeof(AVPacket));

  // Align the buffer to a 64-byte boundary The ffmpeg codec likes this,
  // because it uses SSESSE2.
  _buffer = _buffer_alloc;
  while (((size_t)_buffer) & 31) {
    _buffer += 1;
  }

  fetch_packet();
  _initial_dts = _packet->dts;
  _last_seek = 0;
  _samples_read = 0;
  _buffer_head = 0;
  _buffer_tail = 0;
}

/**
 * xxx
 */
FfmpegAudioCursor::
~FfmpegAudioCursor() {
  cleanup();
}

/**
 * Reset to a standard inactive state.
 */
void FfmpegAudioCursor::
cleanup() {
  if (_frame) {
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 45, 101)
    av_frame_free(&_frame);
#else
    avcodec_free_frame(&_frame);
#endif
    _frame = NULL;
  }

  if (_packet) {
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
    av_packet_free(&_packet);
#else
    if (_packet->data) {
      av_free_packet(_packet);
    }
    delete _packet;
    _packet = NULL;
#endif
  }

  if (_buffer_alloc) {
    delete[] _buffer_alloc;
    _buffer_alloc = 0;
    _buffer = NULL;
  }

  if ((_audio_ctx)&&(_audio_ctx->codec)) {
    avcodec_close(_audio_ctx);
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 52, 0)
    avcodec_free_context(&_audio_ctx);
#else
    delete _audio_ctx;
#endif
  }
  _audio_ctx = NULL;

  if (_format_ctx) {
    _ffvfile.close();
    _format_ctx = NULL;
  }

#ifdef HAVE_SWRESAMPLE
  if (_resample_ctx) {
    swr_free(&_resample_ctx);
    _resample_ctx = NULL;
  }
#endif

  _audio_index = -1;
}

/**
 * Fetches an audio packet and stores it in the packet buffer.  Also sets
 * packet_size and packet_data.
 */
void FfmpegAudioCursor::
fetch_packet() {
  if (_packet->data) {
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
    av_packet_unref(_packet);
#else
    av_free_packet(_packet);
#endif
  }
  while (av_read_frame(_format_ctx, _packet) >= 0) {
    if (_packet->stream_index == _audio_index) {
      _packet_size = _packet->size;
      _packet_data = _packet->data;
      return;
    }
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
    av_packet_unref(_packet);
#else
    av_free_packet(_packet);
#endif
  }
  _packet->data = 0;
  _packet_size = 0;
  _packet_data = 0;
}

/**
 * Reloads the audio buffer by decoding audio packets until one of those audio
 * packets finally yields some samples.  If we encounter the end of the
 * stream, we synthesize silence.
 */
bool FfmpegAudioCursor::
reload_buffer() {
  int got_frame = 0;
  while (!got_frame) {
    // If we're out of packets, generate silence.
    if (_packet->data == nullptr) {
      _buffer_head = 0;
      _buffer_tail = _buffer_size;
      memset(_buffer, 0, _buffer_size * 2);
      return true;
    } else if (_packet_size == 0) {
      fetch_packet();
    }

    AVPacket *pkt;
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
    pkt = av_packet_alloc();
#else
    AVPacket _pkt;
    pkt = &_pkt;
    av_init_packet(pkt);
#endif
    pkt->data = _packet_data;
    pkt->size = _packet_size;

    int len = avcodec_decode_audio4(_audio_ctx, _frame, &got_frame, pkt);
    movies_debug("avcodec_decode_audio4 returned " << len);

#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
    av_packet_free(&pkt);
#else
    av_free_packet(pkt);
#endif

    if (len < 0) {
      return false;
    } else if (len == 0) {
      return true;
    }
    _packet_data += len;
    _packet_size -= len;
  }

  int bufsize;
#ifdef HAVE_SWRESAMPLE
  if (_resample_ctx) {
    // Resample the data to signed 16-bit sample format.
    bufsize = swr_convert(_resample_ctx, (uint8_t **)&_buffer, _buffer_size / 2, (const uint8_t**)_frame->extended_data, _frame->nb_samples);
    bufsize *= _audio_channels * 2;
  } else
#endif
  {
    bufsize = _frame->linesize[0];
    memcpy(_buffer, _frame->data[0], bufsize);
  }
#if LIBAVUTIL_VERSION_INT > AV_VERSION_INT(52, 19, 100)
  av_frame_unref(_frame);
#endif

  if (bufsize > 0) {
    _buffer_head = 0;
    _buffer_tail = (bufsize/2);
    return true;
  }
  return true;
}

/**
 * Seeks to a target location.  Afterward, the packet_time is guaranteed to be
 * less than or equal to the specified time.
 */
void FfmpegAudioCursor::
seek(double t) {
  int64_t target_ts = (int64_t)(t / _audio_timebase);
  if (target_ts < (int64_t)(_initial_dts)) {
    // Attempts to seek before the first packet will fail.
    target_ts = _initial_dts;
  }
  if (av_seek_frame(_format_ctx, _audio_index, target_ts, AVSEEK_FLAG_BACKWARD) < 0) {
    ffmpeg_cat.error() << "Seek failure. Shutting down movie.\n";
    cleanup();
    return;
  }
  avcodec_close(_audio_ctx);
  AVCodec *pAudioCodec = avcodec_find_decoder(_audio_ctx->codec_id);
  if(pAudioCodec == 0) {
    cleanup();
    return;
  }
  if (avcodec_open2(_audio_ctx, pAudioCodec, NULL) < 0) {
    cleanup();
    return;
  }
  _buffer_head = 0;
  _buffer_tail = 0;
  fetch_packet();
  double ts = _packet->dts * _audio_timebase;
  if (t > ts) {
    int skip = (int)((t-ts) * _audio_rate);
    read_samples(skip, 0);
  }
  _last_seek = t;
  _samples_read = 0;
}

/**
 * Read audio samples from the stream.  N is the number of samples you wish to
 * read.  Your buffer must be equal in size to N * channels.  Multiple-channel
 * audio will be interleaved.
 */
void FfmpegAudioCursor::
read_samples(int n, int16_t *data) {
  int desired = n * _audio_channels;

  while (desired > 0) {
    if (_buffer_head == _buffer_tail) {
      if(!reload_buffer()){
        break;
      }
      movies_debug("read_samples() desired samples: " << desired << " N:" << n);
    }
    int available = _buffer_tail - _buffer_head;
    int ncopy = (desired > available) ? available : desired;
    if (ncopy) {
      if (data != 0) {
        memcpy(data, _buffer + _buffer_head, ncopy * 2);
        data += ncopy;
      }
      desired -= ncopy;
      _buffer_head += ncopy;
    }

  }
  _samples_read += n;
}
<commit_msg>ffmpeg: Add async audio decoding loop<commit_after>/**
 * PANDA 3D SOFTWARE
 * Copyright (c) Carnegie Mellon University.  All rights reserved.
 *
 * All use of this software is subject to the terms of the revised BSD
 * license.  You should have received a copy of this license along
 * with this source code in a file named "LICENSE."
 *
 * @file ffmpegAudioCursor.cxx
 * @author jyelon
 * @date 2007-08-01
 */

#include "config_ffmpeg.h"
#include "ffmpegAudioCursor.h"

#include "ffmpegAudio.h"
extern "C" {
  #include "libavutil/dict.h"
  #include "libavutil/opt.h"
  #include "libavcodec/avcodec.h"
  #include "libavformat/avformat.h"
}

#ifdef HAVE_SWRESAMPLE
extern "C" {
  #include "libswresample/swresample.h"
}
#endif

TypeHandle FfmpegAudioCursor::_type_handle;

#if LIBAVFORMAT_VERSION_MAJOR < 53
  #define AVMEDIA_TYPE_AUDIO CODEC_TYPE_AUDIO
#endif

#ifndef AVCODEC_MAX_AUDIO_FRAME_SIZE
// More recent versions of ffmpeg no longer define this.
#define AVCODEC_MAX_AUDIO_FRAME_SIZE 192000
#endif

/**
 * xxx
 */
FfmpegAudioCursor::
FfmpegAudioCursor(FfmpegAudio *src) :
  MovieAudioCursor(src),
  _filename(src->_filename),
  _packet(0),
  _packet_data(0),
  _format_ctx(0),
  _audio_ctx(0),
  _resample_ctx(0),
  _buffer(0),
  _buffer_alloc(0),
  _frame(0)
{
  if (!_ffvfile.open_vfs(_filename)) {
    cleanup();
    return;
  }

  _format_ctx = _ffvfile.get_format_context();
  nassertv(_format_ctx != NULL);

  if (avformat_find_stream_info(_format_ctx, NULL) < 0) {
    cleanup();
    return;
  }

  // As of libavformat version 57.41.100, AVStream.codec is deprecated in favor
  // of AVStream.codecpar.  Fortunately, the two structures have
  // similarly-named members, so we can just switch out the declaration.
#if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 41, 100)
  AVCodecParameters *codecpar;
#else
  AVCodecContext *codecpar;
#endif

  // Find the audio stream
  AVStream *stream = nullptr;
  for (int i = 0; i < (int)_format_ctx->nb_streams; i++) {
#if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 41, 100)
    codecpar = _format_ctx->streams[i]->codecpar;
#else
    codecpar = _format_ctx->streams[i]->codec;
#endif
    if (codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
      _audio_index = i;
      stream = _format_ctx->streams[i];
      break;
    }
  }

  if (stream == nullptr) {
    cleanup();
    return;
  }

  _audio_timebase = av_q2d(stream->time_base);
  _audio_rate = codecpar->sample_rate;
  _audio_channels = codecpar->channels;

  AVCodec *pAudioCodec = avcodec_find_decoder(codecpar->codec_id);
  if (pAudioCodec == 0) {
    cleanup();
    return;
  }

  _audio_ctx = avcodec_alloc_context3(pAudioCodec);

  if (_audio_ctx == nullptr) {
    cleanup();
    return;
  }

#if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 41, 100)
  avcodec_parameters_to_context(_audio_ctx, codecpar);
#else
  avcodec_copy_context(_audio_ctx, codecpar);
#endif

  AVDictionary *opts = NULL;
  av_dict_set(&opts, "request_sample_fmt", "s16", 0);
  if (avcodec_open2(_audio_ctx, pAudioCodec, NULL) < 0) {
    cleanup();
    return;
  }

  av_dict_free(&opts);

  // Set up the resample context if necessary.
  if (_audio_ctx->sample_fmt != AV_SAMPLE_FMT_S16) {
#ifdef HAVE_SWRESAMPLE
    ffmpeg_cat.debug()
      << "Codec does not use signed 16-bit sample format.  Setting up swresample context.\n";

    _resample_ctx = swr_alloc();
    av_opt_set_int(_resample_ctx, "in_channel_count", _audio_channels, 0);
    av_opt_set_int(_resample_ctx, "out_channel_count", _audio_channels, 0);
    av_opt_set_int(_resample_ctx, "in_channel_layout", _audio_ctx->channel_layout, 0);
    av_opt_set_int(_resample_ctx, "out_channel_layout", _audio_ctx->channel_layout, 0);
    av_opt_set_int(_resample_ctx, "in_sample_rate", _audio_ctx->sample_rate, 0);
    av_opt_set_int(_resample_ctx, "out_sample_rate", _audio_ctx->sample_rate, 0);
    av_opt_set_sample_fmt(_resample_ctx, "in_sample_fmt", _audio_ctx->sample_fmt, 0);
    av_opt_set_sample_fmt(_resample_ctx, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);

    if (swr_init(_resample_ctx) != 0) {
      ffmpeg_cat.error()
        << "Failed to set up resample context.\n";
      _resample_ctx = NULL;
    }
#else
    ffmpeg_cat.error()
      << "Codec does not use signed 16-bit sample format, but support for libswresample has not been enabled.\n";
#endif
  }

  _length = (_format_ctx->duration * 1.0) / AV_TIME_BASE;
  _can_seek = true;
  _can_seek_fast = true;

#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 45, 101)
  _frame = av_frame_alloc();
#else
  _frame = avcodec_alloc_frame();
#endif

#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
  _packet = av_packet_alloc();
#else
  _packet = new AVPacket;
#endif

  _buffer_size = AVCODEC_MAX_AUDIO_FRAME_SIZE / 2;
  _buffer_alloc = new int16_t[_buffer_size + 64];

  // Allocate enough space for 1024 samples per channel.
  if ((_packet == 0)||(_buffer_alloc == 0)) {
    cleanup();
    return;
  }
  memset(_packet, 0, sizeof(AVPacket));

  // Align the buffer to a 64-byte boundary The ffmpeg codec likes this,
  // because it uses SSESSE2.
  _buffer = _buffer_alloc;
  while (((size_t)_buffer) & 31) {
    _buffer += 1;
  }

  fetch_packet();
  _initial_dts = _packet->dts;
  _last_seek = 0;
  _samples_read = 0;
  _buffer_head = 0;
  _buffer_tail = 0;
}

/**
 * xxx
 */
FfmpegAudioCursor::
~FfmpegAudioCursor() {
  cleanup();
}

/**
 * Reset to a standard inactive state.
 */
void FfmpegAudioCursor::
cleanup() {
  if (_frame) {
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 45, 101)
    av_frame_free(&_frame);
#else
    avcodec_free_frame(&_frame);
#endif
    _frame = NULL;
  }

  if (_packet) {
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
    av_packet_free(&_packet);
#else
    if (_packet->data) {
      av_free_packet(_packet);
    }
    delete _packet;
    _packet = NULL;
#endif
  }

  if (_buffer_alloc) {
    delete[] _buffer_alloc;
    _buffer_alloc = 0;
    _buffer = NULL;
  }

  if ((_audio_ctx)&&(_audio_ctx->codec)) {
    avcodec_close(_audio_ctx);
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 52, 0)
    avcodec_free_context(&_audio_ctx);
#else
    delete _audio_ctx;
#endif
  }
  _audio_ctx = NULL;

  if (_format_ctx) {
    _ffvfile.close();
    _format_ctx = NULL;
  }

#ifdef HAVE_SWRESAMPLE
  if (_resample_ctx) {
    swr_free(&_resample_ctx);
    _resample_ctx = NULL;
  }
#endif

  _audio_index = -1;
}

/**
 * Fetches an audio packet and stores it in the packet buffer.  Also sets
 * packet_size and packet_data.
 */
void FfmpegAudioCursor::
fetch_packet() {
  if (_packet->data) {
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
    av_packet_unref(_packet);
#else
    av_free_packet(_packet);
#endif
  }
  while (av_read_frame(_format_ctx, _packet) >= 0) {
    if (_packet->stream_index == _audio_index) {
      _packet_size = _packet->size;
      _packet_data = _packet->data;
      return;
    }
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
    av_packet_unref(_packet);
#else
    av_free_packet(_packet);
#endif
  }
  _packet->data = 0;
  _packet_size = 0;
  _packet_data = 0;
}

/**
 * Reloads the audio buffer by decoding audio packets until one of those audio
 * packets finally yields some samples.  If we encounter the end of the
 * stream, we synthesize silence.
 */
bool FfmpegAudioCursor::
reload_buffer() {
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 37, 100)
  // lavc >= 57.37.100 deprecates the old (avcodec_decode_audio*) API in favor
  // of a newer, asynchronous API.  This is great for our purposes - it gives
  // the codec the opportunity to decode in the background (e.g. in another
  // thread or on a dedicated hardware coprocessor).

  // First, let's fill the codec's input buffer with as many packets as it'll
  // take:
  int ret;
  while (_packet->data != nullptr) {
    ret = avcodec_send_packet(_audio_ctx, _packet);

    if (ret != 0) {
      // Nonzero return code is an error.
      break;
    }

    // If we got here, the codec took the packet!  Fetch another one.
    fetch_packet();
    if (_packet->data == nullptr) {
      // fetch_packet() says we're out of packets.  Let the codec know.
      ret = avcodec_send_packet(_audio_ctx, NULL);
    }
  }

  // Expected ret codes are 0 (we ran out of packets) and EAGAIN (codec full)
  if ((ret != 0) && (ret != AVERROR(EAGAIN))) {
    // Some odd error happened.  We can't proceed.
    ffmpeg_cat.error()
      << "avcodec_send_packet returned " << ret << "\n";
    return false;
  }

  // Now we retrieve our frame!
  ret = avcodec_receive_frame(_audio_ctx, _frame);

  if (ret == AVERROR_EOF) {
    // The only way for this to happen is if we're out of packets.
    nassertr(_packet->data == nullptr, false);

    // Synthesize silence:
    _buffer_head = 0;
    _buffer_tail = _buffer_size;
    memset(_buffer, 0, _buffer_size * 2);
    return true;

  } else if (ret != 0) {
    // Some odd error happened.  We can't proceed.
    ffmpeg_cat.error()
      << "avcodec_receive_frame returned " << ret << "\n";
    return false;
  }

  // We now have _frame.  It will be handled below.

#else
  int got_frame = 0;
  while (!got_frame) {
    // If we're out of packets, generate silence.
    if (_packet->data == nullptr) {
      _buffer_head = 0;
      _buffer_tail = _buffer_size;
      memset(_buffer, 0, _buffer_size * 2);
      return true;
    } else if (_packet_size == 0) {
      fetch_packet();
    }

    AVPacket *pkt;
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
    pkt = av_packet_alloc();
#else
    AVPacket _pkt;
    pkt = &_pkt;
    av_init_packet(pkt);
#endif
    pkt->data = _packet_data;
    pkt->size = _packet_size;

    int len = avcodec_decode_audio4(_audio_ctx, _frame, &got_frame, pkt);
    movies_debug("avcodec_decode_audio4 returned " << len);

#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
    av_packet_free(&pkt);
#else
    av_free_packet(pkt);
#endif

    if (len < 0) {
      return false;
    } else if (len == 0) {
      return true;
    }
    _packet_data += len;
    _packet_size -= len;
  }
#endif

  int bufsize;
#ifdef HAVE_SWRESAMPLE
  if (_resample_ctx) {
    // Resample the data to signed 16-bit sample format.
    bufsize = swr_convert(_resample_ctx, (uint8_t **)&_buffer, _buffer_size / 2, (const uint8_t**)_frame->extended_data, _frame->nb_samples);
    bufsize *= _audio_channels * 2;
  } else
#endif
  {
    bufsize = _frame->linesize[0];
    memcpy(_buffer, _frame->data[0], bufsize);
  }
#if LIBAVUTIL_VERSION_INT > AV_VERSION_INT(52, 19, 100)
  av_frame_unref(_frame);
#endif

  if (bufsize > 0) {
    _buffer_head = 0;
    _buffer_tail = (bufsize/2);
    return true;
  }
  return true;
}

/**
 * Seeks to a target location.  Afterward, the packet_time is guaranteed to be
 * less than or equal to the specified time.
 */
void FfmpegAudioCursor::
seek(double t) {
  int64_t target_ts = (int64_t)(t / _audio_timebase);
  if (target_ts < (int64_t)(_initial_dts)) {
    // Attempts to seek before the first packet will fail.
    target_ts = _initial_dts;
  }
  if (av_seek_frame(_format_ctx, _audio_index, target_ts, AVSEEK_FLAG_BACKWARD) < 0) {
    ffmpeg_cat.error() << "Seek failure. Shutting down movie.\n";
    cleanup();
    return;
  }
  avcodec_close(_audio_ctx);
  AVCodec *pAudioCodec = avcodec_find_decoder(_audio_ctx->codec_id);
  if(pAudioCodec == 0) {
    cleanup();
    return;
  }
  if (avcodec_open2(_audio_ctx, pAudioCodec, NULL) < 0) {
    cleanup();
    return;
  }
  _buffer_head = 0;
  _buffer_tail = 0;
  fetch_packet();
  double ts = _packet->dts * _audio_timebase;
  if (t > ts) {
    int skip = (int)((t-ts) * _audio_rate);
    read_samples(skip, 0);
  }
  _last_seek = t;
  _samples_read = 0;
}

/**
 * Read audio samples from the stream.  N is the number of samples you wish to
 * read.  Your buffer must be equal in size to N * channels.  Multiple-channel
 * audio will be interleaved.
 */
void FfmpegAudioCursor::
read_samples(int n, int16_t *data) {
  int desired = n * _audio_channels;

  while (desired > 0) {
    if (_buffer_head == _buffer_tail) {
      if(!reload_buffer()){
        break;
      }
      movies_debug("read_samples() desired samples: " << desired << " N:" << n);
    }
    int available = _buffer_tail - _buffer_head;
    int ncopy = (desired > available) ? available : desired;
    if (ncopy) {
      if (data != 0) {
        memcpy(data, _buffer + _buffer_head, ncopy * 2);
        data += ncopy;
      }
      desired -= ncopy;
      _buffer_head += ncopy;
    }

  }
  _samples_read += n;
}
<|endoftext|>


<commit_before>#include "common/harness.h"
#include "commands/trigger.h"
#include "parser/pg_trigger.h"
#include "parser/postgresparser.h"
#include "planner/create_plan.h"

namespace peloton {
namespace test {

class TriggerTests : public PelotonTest {};

TEST_F(TriggerTests, BasicTest) {
  // Create statement
  auto parser = parser::PostgresParser::GetInstance();

  std::string query1 =
      "CREATE TRIGGER check_update "
      "BEFORE UPDATE OF balance ON accounts "
      "FOR EACH ROW "
      "WHEN (OLD.balance <> NEW.balance) "
      "EXECUTE PROCEDURE check_account_update();";
  auto stmt_list1 = parser.BuildParseTree(query1).release();
  EXPECT_TRUE(stmt_list1->is_valid);
  EXPECT_EQ(StatementType::CREATE, stmt_list1->GetStatement(0)->GetType());
  auto create_trigger_stmt1 =
      static_cast<parser::CreateStatement *>(stmt_list1->GetStatement(0));

  // Create plans
  const planner::CreatePlan plan1(create_trigger_stmt1);

  commands::Trigger trigger1(plan1);
  EXPECT_EQ("check_update", trigger1.GetTriggerName());
  int16_t trigger_type1 = trigger1.GetTriggerType();
  EXPECT_TRUE(TRIGGER_FOR_ROW(trigger_type1));
  EXPECT_TRUE(TRIGGER_FOR_BEFORE(trigger_type1));
  EXPECT_TRUE(TRIGGER_FOR_UPDATE(trigger_type1));
  EXPECT_FALSE(TRIGGER_FOR_DELETE(trigger_type1));

  std::string query2 =
      "CREATE TRIGGER check_update_and_delete "
      "BEFORE UPDATE OF balance OR DELETE ON accounts "
      "FOR EACH ROW "
      "WHEN (OLD.balance <> NEW.balance) "
      "EXECUTE PROCEDURE check_account_update();";
  auto stmt_list2 = parser.BuildParseTree(query2).release();
  EXPECT_TRUE(stmt_list2->is_valid);
  auto create_trigger_stmt2 =
      static_cast<parser::CreateStatement *>(stmt_list2->GetStatement(0));
  const planner::CreatePlan plan2(create_trigger_stmt2);
  commands::Trigger trigger2(plan2);
  EXPECT_EQ("check_update_and_delete", trigger2.GetTriggerName());
  int16_t trigger_type2 = trigger2.GetTriggerType();
  EXPECT_TRUE(TRIGGER_FOR_ROW(trigger_type2));
  EXPECT_TRUE(TRIGGER_FOR_BEFORE(trigger_type2));
  EXPECT_TRUE(TRIGGER_FOR_UPDATE(trigger_type2));
  EXPECT_TRUE(TRIGGER_FOR_DELETE(trigger_type2));

  commands::TriggerList trigger_list;
  trigger_list.AddTrigger(trigger1);
  EXPECT_EQ(1, trigger_list.GetTriggerListSize());
  EXPECT_TRUE(trigger_list.HasTriggerType(
      commands::EnumTriggerType::BEFORE_UPDATE_ROW));
  EXPECT_FALSE(trigger_list.HasTriggerType(
      commands::EnumTriggerType::BEFORE_DELETE_ROW));
  EXPECT_FALSE(trigger_list.HasTriggerType(
      commands::EnumTriggerType::BEFORE_INSERT_ROW));

  trigger_list.AddTrigger(trigger2);
  EXPECT_EQ(2, trigger_list.GetTriggerListSize());
  EXPECT_TRUE(trigger_list.HasTriggerType(
      commands::EnumTriggerType::BEFORE_UPDATE_ROW));
  EXPECT_TRUE(trigger_list.HasTriggerType(
      commands::EnumTriggerType::BEFORE_DELETE_ROW));
  EXPECT_FALSE(trigger_list.HasTriggerType(
      commands::EnumTriggerType::BEFORE_INSERT_ROW));
  LOG_INFO("before test ExecBSInsertTriggers");
  trigger_list.ExecBRInsertTriggers(nullptr);
  // TODO: the effect of this operation should be verified
  // automatically. We have check the log output manually.
  // Auto check can be implemented after UDF is implemented

  delete stmt_list1;
  delete stmt_list2;

}
}
}
<commit_msg>Add unit test for invoking triggers (and helper functions)<commit_after>//===----------------------------------------------------------------------===//
//
//                         Peloton
//
// trigger_test.cpp
//
// Identification: test/commands/trigger_test.cpp
//
// Copyright (c) 2015-17, Carnegie Mellon University Database Group
//
//===----------------------------------------------------------------------===//

#include "catalog/catalog.h"
#include "common/harness.h"
#include "commands/trigger.h"
#include "executor/create_executor.h"
#include "parser/pg_trigger.h"
#include "parser/postgresparser.h"
#include "planner/create_plan.h"

namespace peloton {
namespace test {

class TriggerTests : public PelotonTest {
 protected:
  std::string table_name = "accounts";
  std::string col_1 = "dept_id";
  std::string col_2 = "dept_name";

  // helper for c_str copy
  static char *cstrdup(const char *c_str) {
    char *new_str = new char[strlen(c_str) + 1];
    strcpy(new_str, c_str);
    return new_str;
  }

  void CreateTableHelper() {
    auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance();
    auto txn = txn_manager.BeginTransaction();
    catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn);

    // Insert a table first
    auto id_column = catalog::Column(type::Type::INTEGER,
                                     type::Type::GetTypeSize(type::Type::INTEGER),
                                     col_1, true);
    auto name_column =
      catalog::Column(type::Type::VARCHAR, 32, col_2, false);

    // Schema
    std::unique_ptr<catalog::Schema> table_schema(
      new catalog::Schema({id_column, name_column}));

    std::unique_ptr<executor::ExecutorContext> context(
      new executor::ExecutorContext(txn));

    // Create plans
    planner::CreatePlan node(table_name, DEFAULT_DB_NAME, std::move(table_schema),
                             CreateType::TABLE);

    // Create executer
    executor::CreateExecutor executor(&node, context.get());

    executor.Init();
    executor.Execute();

    txn_manager.CommitTransaction(txn);
  }

  void InsertTupleHelper(int number, std::string text) {
    auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance();
    auto txn = txn_manager.BeginTransaction();

    char *table_name_c = cstrdup(table_name.c_str());
    char *col_1_c = cstrdup(col_1.c_str());
    char *col_2_c = cstrdup(col_2.c_str());

    auto table = catalog::Catalog::GetInstance()->GetTableWithName(
      DEFAULT_DB_NAME, std::string(table_name));

    std::unique_ptr<executor::ExecutorContext> context(
      new executor::ExecutorContext(txn));

    std::unique_ptr<parser::InsertStatement> insert_node(
      new parser::InsertStatement(InsertType::VALUES));

    auto table_ref = new parser::TableRef(TableReferenceType::NAME);
    parser::TableInfo *table_info = new parser::TableInfo();
    table_info->table_name = table_name_c;
    table_ref->table_info_ = table_info;
    insert_node->table_ref_ = table_ref;

    insert_node->columns = new std::vector<char *>;
    insert_node->columns->push_back(col_1_c);
    insert_node->columns->push_back(col_2_c);

    insert_node->insert_values =
      new std::vector<std::vector<expression::AbstractExpression *> *>;
    auto values_ptr = new std::vector<expression::AbstractExpression *>;
    insert_node->insert_values->push_back(values_ptr);

    values_ptr->push_back(new expression::ConstantValueExpression(
      type::ValueFactory::GetIntegerValue(number)));

    values_ptr->push_back(new expression::ConstantValueExpression(
      type::ValueFactory::GetVarcharValue(text)));

    insert_node->select = new parser::SelectStatement();

    planner::InsertPlan node(table, insert_node->columns,
                             insert_node->insert_values);
    executor::InsertExecutor executor(&node, context.get());

    EXPECT_TRUE(executor.Init());
    EXPECT_TRUE(executor.Execute());
    EXPECT_EQ(1, table->GetTupleCount());

    txn_manager.CommitTransaction(txn);
  }
};

TEST_F(TriggerTests, BasicTest) {
  // Create statement
  auto parser = parser::PostgresParser::GetInstance();

  std::string query1 =
      "CREATE TRIGGER check_update "
      "BEFORE UPDATE OF balance ON accounts "
      "FOR EACH ROW "
      "WHEN (OLD.balance <> NEW.balance) "
      "EXECUTE PROCEDURE check_account_update();";
  std::unique_ptr<parser::SQLStatementList> stmt_list1(parser.BuildParseTree(query1).release());
  EXPECT_TRUE(stmt_list1->is_valid);
  EXPECT_EQ(StatementType::CREATE, stmt_list1->GetStatement(0)->GetType());
  auto create_trigger_stmt1 =
      static_cast<parser::CreateStatement *>(stmt_list1->GetStatement(0));

  // Create plans
  const planner::CreatePlan plan1(create_trigger_stmt1);

  commands::Trigger trigger1(plan1);
  EXPECT_EQ("check_update", trigger1.GetTriggerName());
  int16_t trigger_type1 = trigger1.GetTriggerType();
  EXPECT_TRUE(TRIGGER_FOR_ROW(trigger_type1));
  EXPECT_TRUE(TRIGGER_FOR_BEFORE(trigger_type1));
  EXPECT_TRUE(TRIGGER_FOR_UPDATE(trigger_type1));
  EXPECT_FALSE(TRIGGER_FOR_DELETE(trigger_type1));

  std::string query2 =
      "CREATE TRIGGER check_update_and_delete "
      "BEFORE UPDATE OF balance OR DELETE ON accounts "
      "FOR EACH ROW "
      "WHEN (OLD.balance <> NEW.balance) "
      "EXECUTE PROCEDURE check_account_update();";
  std::unique_ptr<parser::SQLStatementList> stmt_list2(parser.BuildParseTree(query2).release());
  EXPECT_TRUE(stmt_list2->is_valid);
  auto create_trigger_stmt2 =
      static_cast<parser::CreateStatement *>(stmt_list2->GetStatement(0));
  const planner::CreatePlan plan2(create_trigger_stmt2);
  commands::Trigger trigger2(plan2);
  EXPECT_EQ("check_update_and_delete", trigger2.GetTriggerName());
  int16_t trigger_type2 = trigger2.GetTriggerType();
  EXPECT_TRUE(TRIGGER_FOR_ROW(trigger_type2));
  EXPECT_TRUE(TRIGGER_FOR_BEFORE(trigger_type2));
  EXPECT_TRUE(TRIGGER_FOR_UPDATE(trigger_type2));
  EXPECT_TRUE(TRIGGER_FOR_DELETE(trigger_type2));

  commands::TriggerList trigger_list;
  trigger_list.AddTrigger(trigger1);
  EXPECT_EQ(1, trigger_list.GetTriggerListSize());
  EXPECT_TRUE(trigger_list.HasTriggerType(
      commands::EnumTriggerType::BEFORE_UPDATE_ROW));
  EXPECT_FALSE(trigger_list.HasTriggerType(
      commands::EnumTriggerType::BEFORE_DELETE_ROW));
  EXPECT_FALSE(trigger_list.HasTriggerType(
      commands::EnumTriggerType::BEFORE_INSERT_ROW));

  trigger_list.AddTrigger(trigger2);
  EXPECT_EQ(2, trigger_list.GetTriggerListSize());
  EXPECT_TRUE(trigger_list.HasTriggerType(
      commands::EnumTriggerType::BEFORE_UPDATE_ROW));
  EXPECT_TRUE(trigger_list.HasTriggerType(
      commands::EnumTriggerType::BEFORE_DELETE_ROW));
  EXPECT_FALSE(trigger_list.HasTriggerType(
      commands::EnumTriggerType::BEFORE_INSERT_ROW));

}

// Test trigger type: before, each row, insert
TEST_F(TriggerTests, BRInsertTriggers) {
  // Bootstrap
  auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance();
  auto parser = parser::PostgresParser::GetInstance();
  catalog::Catalog::GetInstance()->Bootstrap();

  // Create table
  CreateTableHelper();

  // Create statement

  std::string query =
    "CREATE TRIGGER b_r_insert_trigger "
      "BEFORE UPDATE OF dept_id ON accounts "
      "FOR EACH ROW "
      "EXECUTE PROCEDURE b_r_insert_trigger_func();";
  std::unique_ptr<parser::SQLStatementList> stmt_list(parser.BuildParseTree(query).release());
  EXPECT_TRUE(stmt_list->is_valid);
  EXPECT_EQ(StatementType::CREATE, stmt_list->GetStatement(0)->GetType());
  auto create_trigger_stmt =
    static_cast<parser::CreateStatement *>(stmt_list->GetStatement(0));

  // Create plans
  planner::CreatePlan plan(create_trigger_stmt);

  // plan type
  EXPECT_EQ(CreateType::TRIGGER, plan.GetCreateType());

  // type (level, timing, event)
  auto trigger_type = plan.GetTriggerType();
  // level
  EXPECT_TRUE(TRIGGER_FOR_ROW(trigger_type));
  // timing
  EXPECT_TRUE(TRIGGER_FOR_BEFORE(trigger_type));
  EXPECT_FALSE(TRIGGER_FOR_AFTER(trigger_type));
  EXPECT_FALSE(TRIGGER_FOR_INSTEAD(trigger_type));
  // event
  EXPECT_TRUE(TRIGGER_FOR_UPDATE(trigger_type));
  EXPECT_FALSE(TRIGGER_FOR_INSERT(trigger_type));
  EXPECT_FALSE(TRIGGER_FOR_DELETE(trigger_type));
  EXPECT_FALSE(TRIGGER_FOR_TRUNCATE(trigger_type));

  // Execute the create trigger
  auto txn = txn_manager.BeginTransaction();
  std::unique_ptr<executor::ExecutorContext> context2(
    new executor::ExecutorContext(txn));
  executor::CreateExecutor createTriggerExecutor(&plan, context2.get());
  createTriggerExecutor.Init();
  createTriggerExecutor.Execute();
  txn_manager.CommitTransaction(txn);

  // Check the effect of creation
  storage::DataTable *target_table =
    catalog::Catalog::GetInstance()->GetTableWithName(DEFAULT_DB_NAME,
                                                      "accounts");
  EXPECT_EQ(1, target_table->GetTriggerNumber());
  commands::Trigger *new_trigger = target_table->GetTriggerByIndex(0);
  EXPECT_EQ(new_trigger->GetTriggerName(), "b_r_insert_trigger");

  commands::TriggerList *new_trigger_list = target_table->GetTriggerList();
  EXPECT_EQ(1, new_trigger_list->GetTriggerListSize());
  EXPECT_TRUE(new_trigger_list->HasTriggerType(commands::EnumTriggerType::BEFORE_UPDATE_ROW));

  InsertTupleHelper(2333, "LTI");

  // TODO: the effect of this operation should be verified automatically.
  // The UDF should be called after this operation happens.
  // Auto check can be implemented after UDF is implemented.

  // free the database just created
  txn = txn_manager.BeginTransaction();
  catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn);
  txn_manager.CommitTransaction(txn);

}
}
}
<|endoftext|>


<commit_before>//
// Created by haohanwang on 2/24/16.
//

#include "AdaMultiLasso.hpp"

using namespace std;

AdaMultiLasso::AdaMultiLasso() {
    lambda1 = 0;
    lambda2 = 0;
    initTrainingFlag = false;
}

void AdaMultiLasso::setLambda1(double d) {
    lambda1 = d;
}

void AdaMultiLasso::setLambda2(double d) {
    lambda2 = d;
}

void AdaMultiLasso::setSnpsFeature1(MatrixXd xd) {
    snpsFeature1 = xd;
}

void AdaMultiLasso::setSnpsFeature2(MatrixXd xd) {
    snpsFeature2 = xd;
}

MatrixXd AdaMultiLasso::getSnpsFeature1() {
    return snpsFeature1;
}

MatrixXd AdaMultiLasso::getSnpsFeature2() {
    return snpsFeature2;
}

VectorXd AdaMultiLasso::getW() {
    return w;
}

VectorXd AdaMultiLasso::getV() {
    return v;
}

VectorXd AdaMultiLasso::gradient_w() {
    long c = snpsFeature1.cols();
    long k = beta.cols();
    long s = theta.size();
    updateTheta();
    VectorXd grad = VectorXd::Zero(c);
    for (long j=0;j<c;j++){
        grad(j) += (-k*snpsFeature1.row(j).array()/theta.array()).sum();
        grad(j) += (snpsFeature1.row(j).transpose()*(beta.array().abs().matrix())).array().sum();
    }
    return grad;
}

VectorXd AdaMultiLasso::gradient_v() {
    long c = snpsFeature2.cols();
    long k = beta.cols();
    long s = rho.size();
    updateRho();
    VectorXd grad = VectorXd::Zero(c);
    for (long j=0;j<c;j++){
        grad(j) += (-k*snpsFeature1.row(j).array()/theta.array()).sum();
        grad(j) += (snpsFeature1.row(j).transpose()*(beta.array().abs().matrix())).array().sum();  // double check this shortcut
    }
    return grad;
}

void AdaMultiLasso::updateTheta() {
    long c = snpsFeature1.cols();
    theta = VectorXd::Zero(c);
    for (long j=0; j<c; j++){
        theta(j) = (snpsFeature1.col(j)*w).sum();
    }
}

void AdaMultiLasso::updateRho() {
    long c = snpsFeature2.cols();
    rho = VectorXd::Zero(c);
    for (long j=0; j<c; j++){
        rho(j) = (snpsFeature2.col(j)*v).sum();
    }
}

void AdaMultiLasso::initTheta() {
    w = VectorXd::Zero(snpsFeature1.rows());
    long c = snpsFeature1.cols();
    theta = VectorXd::Zero(c);
    for (long j=0; j<c; j++){
        theta(j) = (snpsFeature1.col(j)*w).sum();
    }
}

void AdaMultiLasso::initRho() {
    v = VectorXd::Zero(snpsFeature2.rows());
    long c = snpsFeature2.cols();
    rho = VectorXd::Zero(c);
    for (long j=0; j<c; j++){
        rho(j) = (snpsFeature2.col(j)*v).sum();
    }
}

VectorXd AdaMultiLasso::getTheta() {
    long c = theta.size();
    long rc = c/taskNum;
    VectorXd tmp = VectorXd::Zero(rc);
    for (long i=0;i<rc;i++){
        tmp(i) = theta(i*taskNum);
    }
    return tmp;
}

VectorXd AdaMultiLasso::getRho() {
    long c = rho.size();
    long rc = c/taskNum;
    VectorXd tmp = VectorXd::Zero(rc);
    for (long i=0;i<rc;i++){
        tmp(i) = rho(i*taskNum);
    }
    return tmp;
}

VectorXd AdaMultiLasso::getTheta_formatted() {
    return theta;
}

VectorXd AdaMultiLasso::getRho_formatted() {
    return rho;
}

MatrixXd AdaMultiLasso::getBeta() {
    long r = beta.rows()/taskNum;
    long c = taskNum;
    MatrixXd tmp = MatrixXd::Zero(r, c);
    for (long i=0;i<r;i++){
        for (long j=0;j<c;j++){
            tmp(i,j) = beta(i*taskNum+j, 0);
        }
    }
    return tmp;
}

MatrixXd AdaMultiLasso::getBeta_formatted() {
    return beta;
}

void AdaMultiLasso::setX(MatrixXd xd) {
    try {
        throw 20;
    }
    catch (int) {
        cerr << "TreeLasso does not support setX() and setY() individually, please use setXY instead";
    }
}

void AdaMultiLasso::setY(MatrixXd xd) {
    try {
        throw 20;
    }
    catch (int) {
        cerr << "TreeLasso does not support setX() and setY() individually, please use setXY instead";
    }
}

void AdaMultiLasso::setXY(MatrixXd m, MatrixXd n) {
    X = m;
    y = n;
}

void AdaMultiLasso::initBeta() {
    long c = X.cols();
    long d = y.cols();
    beta = MatrixXd::Random(c, d);
}

double AdaMultiLasso::cost() {
    initTraining();
    return 0.5*(y - X * beta).squaredNorm() + penalty_cost();
}

double AdaMultiLasso::penalty_cost() {
    MatrixXd rb = getBeta();
    long c = rb.rows();
    double result = 0;
    VectorXd theta = getTheta()*lambda1;
    VectorXd rho = getRho()*lambda2;
    for (long i=0;i<c;i++){
        result += theta(i)*rb.row(i).lpNorm<1>() + rho(i)*rb.row(i).norm();
    }
    return result;
}

void AdaMultiLasso::initTraining() {
    if (!initTrainingFlag){
        initTrainingFlag = true;
        // resize X and Y for single task
        initTheta();
        initRho();
        long n = X.rows();
        long c = X.cols();
        taskNum = y.cols();
        MatrixXd tmpX = MatrixXd::Zero(n*taskNum, c*taskNum);
        MatrixXd tmpY = MatrixXd::Zero(n*taskNum, 1);
        VectorXd tmpT = VectorXd::Zero(c*taskNum);
        VectorXd tmpR = VectorXd::Zero(c*taskNum);
        MatrixXd tmpS1 = MatrixXd::Zero(c*taskNum, snpsFeature1.cols());
        MatrixXd tmpS2 = MatrixXd::Zero(c*taskNum, snpsFeature2.cols());
        for (long i=0;i<n;i++){
            for (long j=0;j<taskNum;j++){
                for (long k=0;k<c;k++){
                    tmpX(i*taskNum+j, k*taskNum+j) = X(i, k);
                }
                tmpY(i*taskNum+j, 0) = y(i, j);
                tmpT(i*taskNum+j) = theta(i);
                tmpR(i*taskNum+j) = rho(i);

                tmpS1.row(i*taskNum+j) = snpsFeature1.row(i);
                tmpS2.row(i*taskNum+j) = snpsFeature2.row(i);
            }
        }
        X = tmpX;
        y = tmpY;
        theta = tmpT;
        rho = tmpR;
        snpsFeature1 = tmpS1;
        snpsFeature2 = tmpS2;
        initBeta();
        initC();
        L = ((X.transpose()*X).eigenvalues()).real().maxCoeff() + C.norm()/mu;
    }
}

void AdaMultiLasso::initC() {
    long c = X.cols();
    C = MatrixXd::Zero(c * taskNum, c);
    for (long i = 0; i < c; i++) {
        for (long j = 0; j < taskNum; j++) {
            C(i*j+j, i) = rho(i)*lambda2;
        }
    }
}

MatrixXd AdaMultiLasso::proximal_derivative() {
    long r = beta.rows();
    long c = beta.cols();
    MatrixXd A = MatrixXd::Zero(r*taskNum, c);
    MatrixXd tmp = C*beta;
    for (long i=0;i<r*taskNum;i++){
        A.row(i) = Math::getInstance().L2Thresholding(tmp.row(i)/mu);
    }
    return X.transpose()*(X*beta-y) + C.transpose()*A;
}

MatrixXd AdaMultiLasso::proximal_operator(MatrixXd in, float lr) {
    MatrixXd sign = ((in.array()>0).matrix()).cast<double>();
    sign += -1.0*((in.array()<0).matrix()).cast<double>();
    in = ((in.array().abs()-lr*lambda1*theta.array()).max(0)).matrix();
    return (in.array()*sign.array()).matrix();
}

double AdaMultiLasso::getL() {
    return L;
}

VectorXd AdaMultiLasso::projection(VectorXd in) {
    return NULL;  // could be solved as an optimization problem, let's try to find a close form solution first.
}
<commit_msg>simplest method for projection, probably need update<commit_after>//
// Created by haohanwang on 2/24/16.
//

#include "AdaMultiLasso.hpp"

using namespace std;

AdaMultiLasso::AdaMultiLasso() {
    lambda1 = 0;
    lambda2 = 0;
    initTrainingFlag = false;
}

void AdaMultiLasso::setLambda1(double d) {
    lambda1 = d;
}

void AdaMultiLasso::setLambda2(double d) {
    lambda2 = d;
}

void AdaMultiLasso::setSnpsFeature1(MatrixXd xd) {
    snpsFeature1 = xd;
}

void AdaMultiLasso::setSnpsFeature2(MatrixXd xd) {
    snpsFeature2 = xd;
}

MatrixXd AdaMultiLasso::getSnpsFeature1() {
    return snpsFeature1;
}

MatrixXd AdaMultiLasso::getSnpsFeature2() {
    return snpsFeature2;
}

VectorXd AdaMultiLasso::getW() {
    return w;
}

VectorXd AdaMultiLasso::getV() {
    return v;
}

VectorXd AdaMultiLasso::gradient_w() {
    long c = snpsFeature1.cols();
    long k = beta.cols();
    long s = theta.size();
    updateTheta();
    VectorXd grad = VectorXd::Zero(c);
    for (long j=0;j<c;j++){
        grad(j) += (-k*snpsFeature1.row(j).array()/theta.array()).sum();
        grad(j) += (snpsFeature1.row(j).transpose()*(beta.array().abs().matrix())).array().sum();
    }
    return grad;
}

VectorXd AdaMultiLasso::gradient_v() {
    long c = snpsFeature2.cols();
    long k = beta.cols();
    long s = rho.size();
    updateRho();
    VectorXd grad = VectorXd::Zero(c);
    for (long j=0;j<c;j++){
        grad(j) += (-k*snpsFeature1.row(j).array()/theta.array()).sum();
        grad(j) += (snpsFeature1.row(j).transpose()*(beta.array().abs().matrix())).array().sum();  // double check this shortcut
    }
    return grad;
}

void AdaMultiLasso::updateTheta() {
    long c = snpsFeature1.cols();
    theta = VectorXd::Zero(c);
    for (long j=0; j<c; j++){
        theta(j) = (snpsFeature1.col(j)*w).sum();
    }
}

void AdaMultiLasso::updateRho() {
    long c = snpsFeature2.cols();
    rho = VectorXd::Zero(c);
    for (long j=0; j<c; j++){
        rho(j) = (snpsFeature2.col(j)*v).sum();
    }
}

void AdaMultiLasso::initTheta() {
    w = VectorXd::Zero(snpsFeature1.rows());
    long c = snpsFeature1.cols();
    theta = VectorXd::Zero(c);
    for (long j=0; j<c; j++){
        theta(j) = (snpsFeature1.col(j)*w).sum();
    }
}

void AdaMultiLasso::initRho() {
    v = VectorXd::Zero(snpsFeature2.rows());
    long c = snpsFeature2.cols();
    rho = VectorXd::Zero(c);
    for (long j=0; j<c; j++){
        rho(j) = (snpsFeature2.col(j)*v).sum();
    }
}

VectorXd AdaMultiLasso::getTheta() {
    long c = theta.size();
    long rc = c/taskNum;
    VectorXd tmp = VectorXd::Zero(rc);
    for (long i=0;i<rc;i++){
        tmp(i) = theta(i*taskNum);
    }
    return tmp;
}

VectorXd AdaMultiLasso::getRho() {
    long c = rho.size();
    long rc = c/taskNum;
    VectorXd tmp = VectorXd::Zero(rc);
    for (long i=0;i<rc;i++){
        tmp(i) = rho(i*taskNum);
    }
    return tmp;
}

VectorXd AdaMultiLasso::getTheta_formatted() {
    return theta;
}

VectorXd AdaMultiLasso::getRho_formatted() {
    return rho;
}

MatrixXd AdaMultiLasso::getBeta() {
    long r = beta.rows()/taskNum;
    long c = taskNum;
    MatrixXd tmp = MatrixXd::Zero(r, c);
    for (long i=0;i<r;i++){
        for (long j=0;j<c;j++){
            tmp(i,j) = beta(i*taskNum+j, 0);
        }
    }
    return tmp;
}

MatrixXd AdaMultiLasso::getBeta_formatted() {
    return beta;
}

void AdaMultiLasso::setX(MatrixXd xd) {
    try {
        throw 20;
    }
    catch (int) {
        cerr << "TreeLasso does not support setX() and setY() individually, please use setXY instead";
    }
}

void AdaMultiLasso::setY(MatrixXd xd) {
    try {
        throw 20;
    }
    catch (int) {
        cerr << "TreeLasso does not support setX() and setY() individually, please use setXY instead";
    }
}

void AdaMultiLasso::setXY(MatrixXd m, MatrixXd n) {
    X = m;
    y = n;
}

void AdaMultiLasso::initBeta() {
    long c = X.cols();
    long d = y.cols();
    beta = MatrixXd::Random(c, d);
}

double AdaMultiLasso::cost() {
    initTraining();
    return 0.5*(y - X * beta).squaredNorm() + penalty_cost();
}

double AdaMultiLasso::penalty_cost() {
    MatrixXd rb = getBeta();
    long c = rb.rows();
    double result = 0;
    VectorXd theta = getTheta()*lambda1;
    VectorXd rho = getRho()*lambda2;
    for (long i=0;i<c;i++){
        result += theta(i)*rb.row(i).lpNorm<1>() + rho(i)*rb.row(i).norm();
    }
    return result;
}

void AdaMultiLasso::initTraining() {
    if (!initTrainingFlag){
        initTrainingFlag = true;
        // resize X and Y for single task
        initTheta();
        initRho();
        long n = X.rows();
        long c = X.cols();
        taskNum = y.cols();
        MatrixXd tmpX = MatrixXd::Zero(n*taskNum, c*taskNum);
        MatrixXd tmpY = MatrixXd::Zero(n*taskNum, 1);
        VectorXd tmpT = VectorXd::Zero(c*taskNum);
        VectorXd tmpR = VectorXd::Zero(c*taskNum);
        MatrixXd tmpS1 = MatrixXd::Zero(c*taskNum, snpsFeature1.cols());
        MatrixXd tmpS2 = MatrixXd::Zero(c*taskNum, snpsFeature2.cols());
        for (long i=0;i<n;i++){
            for (long j=0;j<taskNum;j++){
                for (long k=0;k<c;k++){
                    tmpX(i*taskNum+j, k*taskNum+j) = X(i, k);
                }
                tmpY(i*taskNum+j, 0) = y(i, j);
                tmpT(i*taskNum+j) = theta(i);
                tmpR(i*taskNum+j) = rho(i);

                tmpS1.row(i*taskNum+j) = snpsFeature1.row(i);
                tmpS2.row(i*taskNum+j) = snpsFeature2.row(i);
            }
        }
        X = tmpX;
        y = tmpY;
        theta = tmpT;
        rho = tmpR;
        snpsFeature1 = tmpS1;
        snpsFeature2 = tmpS2;
        initBeta();
        initC();
        L = ((X.transpose()*X).eigenvalues()).real().maxCoeff() + C.norm()/mu;
    }
}

void AdaMultiLasso::initC() {
    long c = X.cols();
    C = MatrixXd::Zero(c * taskNum, c);
    for (long i = 0; i < c; i++) {
        for (long j = 0; j < taskNum; j++) {
            C(i*j+j, i) = rho(i)*lambda2;
        }
    }
}

MatrixXd AdaMultiLasso::proximal_derivative() {
    long r = beta.rows();
    long c = beta.cols();
    MatrixXd A = MatrixXd::Zero(r*taskNum, c);
    MatrixXd tmp = C*beta;
    for (long i=0;i<r*taskNum;i++){
        A.row(i) = Math::getInstance().L2Thresholding(tmp.row(i)/mu);
    }
    return X.transpose()*(X*beta-y) + C.transpose()*A;
}

MatrixXd AdaMultiLasso::proximal_operator(MatrixXd in, float lr) {
    MatrixXd sign = ((in.array()>0).matrix()).cast<double>();
    sign += -1.0*((in.array()<0).matrix()).cast<double>();
    in = ((in.array().abs()-lr*lambda1*theta.array()).max(0)).matrix();
    return (in.array()*sign.array()).matrix();
}

double AdaMultiLasso::getL() {
    return L;
}

VectorXd AdaMultiLasso::projection(VectorXd in) {
    // solve the projection as a constrained optimization problem, by solving lagrange dual
    long l = in.size();
    VectorXd x = VectorXd::Zero(l);
    MatrixXd Q = MatrixXd::Identity(l,l);
    VectorXd a = VectorXd::Ones(l);
    VectorXd z = VectorXd::Zero(l);
    int steps = 100;
    int i = 0;
    double l1 = 1e-3;
    double l2 = 1e-3;
    double lr = 1e-3;
    VectorXd grad = VectorXd::Zero(l);
    while (i<steps && (x*Q*x.transpose()-in.transpose()-x).sum()>1e-4){
        grad = x*Q-in.transpose()+l1*a-l2*a;
        x = x - lr * grad;
    }
    return x;
}
<|endoftext|>


<commit_before>//=======================================================================
// Copyright Baptiste Wicht 2011.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
//  http://www.boost.org/LICENSE_1_0.txt)
//=======================================================================

#include "mtac/EscapeAnalysis.hpp"
#include "mtac/Utils.hpp"

using namespace eddic;

mtac::EscapedVariables mtac::escape_analysis(std::shared_ptr<mtac::Function> function){
    mtac::EscapedVariables pointer_escaped = std::make_shared<mtac::EscapedVariables::element_type>();

    for(auto& block : function->getBasicBlocks()){
        for(auto& statement : block->statements){
            //Passing a variable as param by address escape its liveness
            if(auto* ptr = boost::get<std::shared_ptr<mtac::Param>>(&statement)){
                auto& param = *ptr;

                if(param->address){
                    if(mtac::isVariable(param->arg)){
                        auto var = boost::get<std::shared_ptr<Variable>>(param->arg);
                        pointer_escaped->insert(var);
                    }
                }
            }
            //Taking the address of a variable escape its liveness
            else if(auto* ptr = boost::get<std::shared_ptr<mtac::Quadruple>>(&statement)){
                auto& quadruple = *ptr;

                if(quadruple->op == mtac::Operator::PASSIGN){
                    if(quadruple->arg1 && mtac::isVariable(*quadruple->arg1)){
                        auto var = boost::get<std::shared_ptr<Variable>>(*quadruple->arg1);
                        pointer_escaped->insert(var);
                    }
                } else if(quadruple->op == mtac::Operator::DOT_PASSIGN){
                    if(quadruple->arg2 && mtac::isVariable(*quadruple->arg2)){
                        auto var = boost::get<std::shared_ptr<Variable>>(*quadruple->arg2);
                        pointer_escaped->insert(var);
                    }
                }
            }
        }
    }

    return pointer_escaped;
}
<commit_msg>Complete escape analysis<commit_after>//=======================================================================
// Copyright Baptiste Wicht 2011.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
//  http://www.boost.org/LICENSE_1_0.txt)
//=======================================================================

#include "mtac/EscapeAnalysis.hpp"
#include "mtac/Utils.hpp"

using namespace eddic;

mtac::EscapedVariables mtac::escape_analysis(std::shared_ptr<mtac::Function> function){
    mtac::EscapedVariables pointer_escaped = std::make_shared<mtac::EscapedVariables::element_type>();

    for(auto& block : function->getBasicBlocks()){
        for(auto& statement : block->statements){
            //Passing a variable as param by address escape its liveness
            if(auto* ptr = boost::get<std::shared_ptr<mtac::Param>>(&statement)){
                auto& param = *ptr;

                if(param->address){
                    if(mtac::isVariable(param->arg)){
                        auto var = boost::get<std::shared_ptr<Variable>>(param->arg);
                        pointer_escaped->insert(var);
                    }
                }
            }
            //Taking the address of a variable escape its liveness
            else if(auto* ptr = boost::get<std::shared_ptr<mtac::Quadruple>>(&statement)){
                auto& quadruple = *ptr;

                if(quadruple->op == mtac::Operator::PASSIGN){
                    if(quadruple->arg1 && mtac::isVariable(*quadruple->arg1)){
                        auto var = boost::get<std::shared_ptr<Variable>>(*quadruple->arg1);
                        pointer_escaped->insert(var);
                    }
                } else if(quadruple->op == mtac::Operator::DOT_PASSIGN){
                    if(quadruple->arg2 && mtac::isVariable(*quadruple->arg2)){
                        auto var = boost::get<std::shared_ptr<Variable>>(*quadruple->arg2);
                        pointer_escaped->insert(var);
                    }
                } else if(quadruple->op == mtac::Operator::PDOT){
                    if(quadruple->arg1 && mtac::isVariable(*quadruple->arg1)){
                        auto var = boost::get<std::shared_ptr<Variable>>(*quadruple->arg1);
                        pointer_escaped->insert(var);
                    }
                }
            }
        }
    }

    return pointer_escaped;
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2019 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <stdlib.h>
#include <sys/system_properties.h>
#include <sys/types.h>
#include <sys/wait.h>

#include <random>

#include "perfetto/base/logging.h"
#include "perfetto/tracing/core/data_source_config.h"
#include "src/base/test/test_task_runner.h"
#include "test/android_test_utils.h"
#include "test/gtest_and_gmock.h"
#include "test/test_helper.h"

#include "protos/perfetto/config/profiling/heapprofd_config.gen.h"
#include "protos/perfetto/trace/profiling/profile_common.gen.h"
#include "protos/perfetto/trace/profiling/profile_packet.gen.h"
#include "protos/perfetto/trace/trace_packet.gen.h"

namespace perfetto {
namespace {

// Size of individual (repeated) allocations done by the test apps (must be kept
// in sync with their sources).
constexpr uint64_t kTestSamplingInterval = 4096;
constexpr uint64_t kExpectedIndividualAllocSz = 4153;
// Tests rely on the sampling behaviour where allocations larger than the
// sampling interval are recorded at their actual size.
static_assert(kExpectedIndividualAllocSz > kTestSamplingInterval,
              "kTestSamplingInterval invalid");

// Activity that runs a JNI thread that repeatedly calls
// malloc(kExpectedIndividualAllocSz).
static char kMallocActivity[] = "MainActivity";
// Activity that runs a java thread that repeatedly constructs small java
// objects.
static char kJavaAllocActivity[] = "JavaAllocActivity";

std::string RandomSessionName() {
  std::random_device rd;
  std::default_random_engine generator(rd());
  std::uniform_int_distribution<char> distribution('a', 'z');

  constexpr size_t kSessionNameLen = 20;
  std::string result(kSessionNameLen, '\0');
  for (size_t i = 0; i < kSessionNameLen; ++i)
    result[i] = distribution(generator);
  return result;
}

// Starts the activity `activity` of the app `app_name` and later starts
// recording a trace with the allocations in `heap_names`.
//
// `heap_names` is a list of the heap names whose allocations will be recorded.
// An empty list means that only the allocations in the default malloc heap
// ("libc.malloc") are recorded.
//
// Returns the recorded trace.
std::vector<protos::gen::TracePacket> ProfileRuntime(
    const std::string& app_name,
    const std::string& activity,
    const std::vector<std::string>& heap_names) {
  base::TestTaskRunner task_runner;

  // (re)start the target app's main activity
  if (IsAppRunning(app_name)) {
    StopApp(app_name, "old.app.stopped", &task_runner);
    task_runner.RunUntilCheckpoint("old.app.stopped", 1000 /*ms*/);
  }
  StartAppActivity(app_name, activity, "target.app.running", &task_runner,
                   /*delay_ms=*/100);
  task_runner.RunUntilCheckpoint("target.app.running", 1000 /*ms*/);

  // set up tracing
  TestHelper helper(&task_runner);
  helper.ConnectConsumer();
  helper.WaitForConsumerConnect();

  TraceConfig trace_config;
  trace_config.add_buffers()->set_size_kb(10 * 1024);
  trace_config.set_duration_ms(4000);
  trace_config.set_unique_session_name(RandomSessionName().c_str());

  auto* ds_config = trace_config.add_data_sources()->mutable_config();
  ds_config->set_name("android.heapprofd");
  ds_config->set_target_buffer(0);

  protos::gen::HeapprofdConfig heapprofd_config;
  heapprofd_config.set_sampling_interval_bytes(kTestSamplingInterval);
  heapprofd_config.add_process_cmdline(app_name.c_str());
  heapprofd_config.set_block_client(true);
  heapprofd_config.set_all(false);
  for (const std::string& heap_name : heap_names) {
    heapprofd_config.add_heaps(heap_name);
  }
  ds_config->set_heapprofd_config_raw(heapprofd_config.SerializeAsString());

  // start tracing
  helper.StartTracing(trace_config);
  helper.WaitForTracingDisabled();
  helper.ReadData();
  helper.WaitForReadData();

  return helper.trace();
}

// Starts recording a trace with the allocations in `heap_names` and later
// starts the activity `activity` of the app `app_name`
//
// `heap_names` is a list of the heap names whose allocations will be recorded.
// An empty list means that only the allocation in the default malloc heap
// ("libc.malloc") are recorded.
//
// Returns the recorded trace.
std::vector<protos::gen::TracePacket> ProfileStartup(
    const std::string& app_name,
    const std::string& activity,
    const std::vector<std::string>& heap_names,
    const bool enable_extra_guardrails = false) {
  base::TestTaskRunner task_runner;

  if (IsAppRunning(app_name)) {
    StopApp(app_name, "old.app.stopped", &task_runner);
    task_runner.RunUntilCheckpoint("old.app.stopped", 1000 /*ms*/);
  }

  // set up tracing
  TestHelper helper(&task_runner);
  helper.ConnectConsumer();
  helper.WaitForConsumerConnect();

  TraceConfig trace_config;
  trace_config.add_buffers()->set_size_kb(10 * 1024);
  trace_config.set_duration_ms(4000);
  trace_config.set_enable_extra_guardrails(enable_extra_guardrails);
  trace_config.set_unique_session_name(RandomSessionName().c_str());

  auto* ds_config = trace_config.add_data_sources()->mutable_config();
  ds_config->set_name("android.heapprofd");
  ds_config->set_target_buffer(0);

  protos::gen::HeapprofdConfig heapprofd_config;
  heapprofd_config.set_sampling_interval_bytes(kTestSamplingInterval);
  heapprofd_config.add_process_cmdline(app_name.c_str());
  heapprofd_config.set_block_client(true);
  heapprofd_config.set_all(false);
  for (const std::string& heap_name : heap_names) {
    heapprofd_config.add_heaps(heap_name);
  }
  ds_config->set_heapprofd_config_raw(heapprofd_config.SerializeAsString());

  // start tracing
  helper.StartTracing(trace_config);

  // start app
  StartAppActivity(app_name, activity, "target.app.running", &task_runner,
                   /*delay_ms=*/100);
  task_runner.RunUntilCheckpoint("target.app.running", 2000 /*ms*/);

  helper.WaitForTracingDisabled();
  helper.ReadData();
  helper.WaitForReadData();

  return helper.trace();
}

// Check that `packets` contain some allocations performed by kMallocActivity.
void AssertExpectedMallocsPresent(
    const std::vector<protos::gen::TracePacket>& packets) {
  ASSERT_GT(packets.size(), 0u);

  // TODO(rsavitski): assert particular stack frames once we clarify the
  // expected behaviour of unwinding native libs within an apk.
  // Until then, look for an allocation that is a multiple of the expected
  // allocation size.
  bool found_alloc = false;
  bool found_proc_dump = false;
  for (const auto& packet : packets) {
    for (const auto& proc_dump : packet.profile_packet().process_dumps()) {
      found_proc_dump = true;
      for (const auto& sample : proc_dump.samples()) {
        if (sample.self_allocated() > 0 &&
            sample.self_allocated() % kExpectedIndividualAllocSz == 0) {
          found_alloc = true;

          EXPECT_TRUE(sample.self_freed() > 0 &&
                      sample.self_freed() % kExpectedIndividualAllocSz == 0)
              << "self_freed: " << sample.self_freed();
        }
      }
    }
  }
  ASSERT_TRUE(found_proc_dump);
  ASSERT_TRUE(found_alloc);
}

// Check that `packets` contain some allocations performed by
// kJavaAllocActivity.
void AssertExpectedArtAllocsPresent(
    const std::vector<protos::gen::TracePacket>& packets) {
  ASSERT_GT(packets.size(), 0u);

  bool found_alloc = false;
  bool found_proc_dump = false;
  for (const auto& packet : packets) {
    for (const auto& proc_dump : packet.profile_packet().process_dumps()) {
      found_proc_dump = true;
      for (const auto& sample : proc_dump.samples()) {
        if (sample.self_allocated() > 0) {
          found_alloc = true;

          // ART only reports when memory is allocated, not when it's released.
          EXPECT_TRUE(sample.self_freed() == 0)
              << "self_freed: " << sample.self_freed();
        }
      }
    }
  }
  ASSERT_TRUE(found_proc_dump);
  ASSERT_TRUE(found_alloc);
}

void AssertNoProfileContents(
    const std::vector<protos::gen::TracePacket>& packets) {
  // If profile packets are present, they must be empty.
  for (const auto& packet : packets) {
    ASSERT_EQ(packet.profile_packet().process_dumps_size(), 0);
  }
}

TEST(HeapprofdCtsTest, DebuggableAppRuntime) {
  std::string app_name = "android.perfetto.cts.app.debuggable";
  const auto& packets =
      ProfileRuntime(app_name, kMallocActivity, /*heap_names=*/{});
  AssertExpectedMallocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, DebuggableAppStartup) {
  std::string app_name = "android.perfetto.cts.app.debuggable";
  const auto& packets =
      ProfileStartup(app_name, kMallocActivity, /*heap_names=*/{});
  AssertExpectedMallocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, ProfileableAppRuntime) {
  std::string app_name = "android.perfetto.cts.app.profileable";
  const auto& packets =
      ProfileRuntime(app_name, kMallocActivity, /*heap_names=*/{});
  AssertExpectedMallocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, ProfileableAppStartup) {
  std::string app_name = "android.perfetto.cts.app.profileable";
  const auto& packets =
      ProfileStartup(app_name, kMallocActivity, /*heap_names=*/{});
  AssertExpectedMallocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, ReleaseAppRuntime) {
  std::string app_name = "android.perfetto.cts.app.release";
  const auto& packets =
      ProfileRuntime(app_name, kMallocActivity, /*heap_names=*/{});

  if (IsUserBuild())
    AssertNoProfileContents(packets);
  else
    AssertExpectedMallocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, ReleaseAppStartup) {
  std::string app_name = "android.perfetto.cts.app.release";
  const auto& packets =
      ProfileStartup(app_name, kMallocActivity, /*heap_names=*/{});

  if (IsUserBuild())
    AssertNoProfileContents(packets);
  else
    AssertExpectedMallocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, NonProfileableAppRuntime) {
  std::string app_name = "android.perfetto.cts.app.nonprofileable";
  const auto& packets =
      ProfileRuntime(app_name, kMallocActivity, /*heap_names=*/{});
  if (IsUserBuild())
    AssertNoProfileContents(packets);
  else
    AssertExpectedMallocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, NonProfileableAppStartup) {
  std::string app_name = "android.perfetto.cts.app.nonprofileable";
  const auto& packets =
      ProfileStartup(app_name, kMallocActivity, /*heap_names=*/{});
  if (IsUserBuild())
    AssertNoProfileContents(packets);
  else
    AssertExpectedMallocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, JavaHeapRuntime) {
  std::string app_name = "android.perfetto.cts.app.debuggable";
  const auto& packets = ProfileRuntime(app_name, kJavaAllocActivity,
                                       /*heap_names=*/{"com.android.art"});
  AssertExpectedArtAllocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, JavaHeapStartup) {
  std::string app_name = "android.perfetto.cts.app.debuggable";
  const auto& packets = ProfileStartup(app_name, kJavaAllocActivity,
                                       /*heap_names=*/{"com.android.art"});
  AssertExpectedArtAllocsPresent(packets);
  StopApp(app_name);
}

}  // namespace
}  // namespace perfetto
<commit_msg>heapprofd_test: Do not check frees for ART heap<commit_after>/*
 * Copyright (C) 2019 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <stdlib.h>
#include <sys/system_properties.h>
#include <sys/types.h>
#include <sys/wait.h>

#include <random>

#include "perfetto/base/logging.h"
#include "perfetto/tracing/core/data_source_config.h"
#include "src/base/test/test_task_runner.h"
#include "test/android_test_utils.h"
#include "test/gtest_and_gmock.h"
#include "test/test_helper.h"

#include "protos/perfetto/config/profiling/heapprofd_config.gen.h"
#include "protos/perfetto/trace/profiling/profile_common.gen.h"
#include "protos/perfetto/trace/profiling/profile_packet.gen.h"
#include "protos/perfetto/trace/trace_packet.gen.h"

namespace perfetto {
namespace {

// Size of individual (repeated) allocations done by the test apps (must be kept
// in sync with their sources).
constexpr uint64_t kTestSamplingInterval = 4096;
constexpr uint64_t kExpectedIndividualAllocSz = 4153;
// Tests rely on the sampling behaviour where allocations larger than the
// sampling interval are recorded at their actual size.
static_assert(kExpectedIndividualAllocSz > kTestSamplingInterval,
              "kTestSamplingInterval invalid");

// Activity that runs a JNI thread that repeatedly calls
// malloc(kExpectedIndividualAllocSz).
static char kMallocActivity[] = "MainActivity";
// Activity that runs a java thread that repeatedly constructs small java
// objects.
static char kJavaAllocActivity[] = "JavaAllocActivity";

std::string RandomSessionName() {
  std::random_device rd;
  std::default_random_engine generator(rd());
  std::uniform_int_distribution<char> distribution('a', 'z');

  constexpr size_t kSessionNameLen = 20;
  std::string result(kSessionNameLen, '\0');
  for (size_t i = 0; i < kSessionNameLen; ++i)
    result[i] = distribution(generator);
  return result;
}

// Starts the activity `activity` of the app `app_name` and later starts
// recording a trace with the allocations in `heap_names`.
//
// `heap_names` is a list of the heap names whose allocations will be recorded.
// An empty list means that only the allocations in the default malloc heap
// ("libc.malloc") are recorded.
//
// Returns the recorded trace.
std::vector<protos::gen::TracePacket> ProfileRuntime(
    const std::string& app_name,
    const std::string& activity,
    const std::vector<std::string>& heap_names) {
  base::TestTaskRunner task_runner;

  // (re)start the target app's main activity
  if (IsAppRunning(app_name)) {
    StopApp(app_name, "old.app.stopped", &task_runner);
    task_runner.RunUntilCheckpoint("old.app.stopped", 1000 /*ms*/);
  }
  StartAppActivity(app_name, activity, "target.app.running", &task_runner,
                   /*delay_ms=*/100);
  task_runner.RunUntilCheckpoint("target.app.running", 1000 /*ms*/);

  // set up tracing
  TestHelper helper(&task_runner);
  helper.ConnectConsumer();
  helper.WaitForConsumerConnect();

  TraceConfig trace_config;
  trace_config.add_buffers()->set_size_kb(10 * 1024);
  trace_config.set_duration_ms(4000);
  trace_config.set_unique_session_name(RandomSessionName().c_str());

  auto* ds_config = trace_config.add_data_sources()->mutable_config();
  ds_config->set_name("android.heapprofd");
  ds_config->set_target_buffer(0);

  protos::gen::HeapprofdConfig heapprofd_config;
  heapprofd_config.set_sampling_interval_bytes(kTestSamplingInterval);
  heapprofd_config.add_process_cmdline(app_name.c_str());
  heapprofd_config.set_block_client(true);
  heapprofd_config.set_all(false);
  for (const std::string& heap_name : heap_names) {
    heapprofd_config.add_heaps(heap_name);
  }
  ds_config->set_heapprofd_config_raw(heapprofd_config.SerializeAsString());

  // start tracing
  helper.StartTracing(trace_config);
  helper.WaitForTracingDisabled();
  helper.ReadData();
  helper.WaitForReadData();

  return helper.trace();
}

// Starts recording a trace with the allocations in `heap_names` and later
// starts the activity `activity` of the app `app_name`
//
// `heap_names` is a list of the heap names whose allocations will be recorded.
// An empty list means that only the allocation in the default malloc heap
// ("libc.malloc") are recorded.
//
// Returns the recorded trace.
std::vector<protos::gen::TracePacket> ProfileStartup(
    const std::string& app_name,
    const std::string& activity,
    const std::vector<std::string>& heap_names,
    const bool enable_extra_guardrails = false) {
  base::TestTaskRunner task_runner;

  if (IsAppRunning(app_name)) {
    StopApp(app_name, "old.app.stopped", &task_runner);
    task_runner.RunUntilCheckpoint("old.app.stopped", 1000 /*ms*/);
  }

  // set up tracing
  TestHelper helper(&task_runner);
  helper.ConnectConsumer();
  helper.WaitForConsumerConnect();

  TraceConfig trace_config;
  trace_config.add_buffers()->set_size_kb(10 * 1024);
  trace_config.set_duration_ms(4000);
  trace_config.set_enable_extra_guardrails(enable_extra_guardrails);
  trace_config.set_unique_session_name(RandomSessionName().c_str());

  auto* ds_config = trace_config.add_data_sources()->mutable_config();
  ds_config->set_name("android.heapprofd");
  ds_config->set_target_buffer(0);

  protos::gen::HeapprofdConfig heapprofd_config;
  heapprofd_config.set_sampling_interval_bytes(kTestSamplingInterval);
  heapprofd_config.add_process_cmdline(app_name.c_str());
  heapprofd_config.set_block_client(true);
  heapprofd_config.set_all(false);
  for (const std::string& heap_name : heap_names) {
    heapprofd_config.add_heaps(heap_name);
  }
  ds_config->set_heapprofd_config_raw(heapprofd_config.SerializeAsString());

  // start tracing
  helper.StartTracing(trace_config);

  // start app
  StartAppActivity(app_name, activity, "target.app.running", &task_runner,
                   /*delay_ms=*/100);
  task_runner.RunUntilCheckpoint("target.app.running", 2000 /*ms*/);

  helper.WaitForTracingDisabled();
  helper.ReadData();
  helper.WaitForReadData();

  return helper.trace();
}

// Check that `packets` contain some allocations performed by kMallocActivity.
void AssertExpectedMallocsPresent(
    const std::vector<protos::gen::TracePacket>& packets) {
  ASSERT_GT(packets.size(), 0u);

  // TODO(rsavitski): assert particular stack frames once we clarify the
  // expected behaviour of unwinding native libs within an apk.
  // Until then, look for an allocation that is a multiple of the expected
  // allocation size.
  bool found_alloc = false;
  bool found_proc_dump = false;
  for (const auto& packet : packets) {
    for (const auto& proc_dump : packet.profile_packet().process_dumps()) {
      found_proc_dump = true;
      for (const auto& sample : proc_dump.samples()) {
        if (sample.self_allocated() > 0 &&
            sample.self_allocated() % kExpectedIndividualAllocSz == 0) {
          found_alloc = true;

          EXPECT_TRUE(sample.self_freed() > 0 &&
                      sample.self_freed() % kExpectedIndividualAllocSz == 0)
              << "self_freed: " << sample.self_freed();
        }
      }
    }
  }
  ASSERT_TRUE(found_proc_dump);
  ASSERT_TRUE(found_alloc);
}

// Check that `packets` contain some allocations performed by
// kJavaAllocActivity.
void AssertExpectedArtAllocsPresent(
    const std::vector<protos::gen::TracePacket>& packets) {
  ASSERT_GT(packets.size(), 0u);

  bool found_alloc = false;
  bool found_proc_dump = false;
  for (const auto& packet : packets) {
    for (const auto& proc_dump : packet.profile_packet().process_dumps()) {
      found_proc_dump = true;
      for (const auto& sample : proc_dump.samples()) {
        if (sample.self_allocated() > 0) {
          found_alloc = true;
        }
      }
    }
  }
  ASSERT_TRUE(found_proc_dump);
  ASSERT_TRUE(found_alloc);
}

void AssertNoProfileContents(
    const std::vector<protos::gen::TracePacket>& packets) {
  // If profile packets are present, they must be empty.
  for (const auto& packet : packets) {
    ASSERT_EQ(packet.profile_packet().process_dumps_size(), 0);
  }
}

TEST(HeapprofdCtsTest, DebuggableAppRuntime) {
  std::string app_name = "android.perfetto.cts.app.debuggable";
  const auto& packets =
      ProfileRuntime(app_name, kMallocActivity, /*heap_names=*/{});
  AssertExpectedMallocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, DebuggableAppStartup) {
  std::string app_name = "android.perfetto.cts.app.debuggable";
  const auto& packets =
      ProfileStartup(app_name, kMallocActivity, /*heap_names=*/{});
  AssertExpectedMallocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, ProfileableAppRuntime) {
  std::string app_name = "android.perfetto.cts.app.profileable";
  const auto& packets =
      ProfileRuntime(app_name, kMallocActivity, /*heap_names=*/{});
  AssertExpectedMallocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, ProfileableAppStartup) {
  std::string app_name = "android.perfetto.cts.app.profileable";
  const auto& packets =
      ProfileStartup(app_name, kMallocActivity, /*heap_names=*/{});
  AssertExpectedMallocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, ReleaseAppRuntime) {
  std::string app_name = "android.perfetto.cts.app.release";
  const auto& packets =
      ProfileRuntime(app_name, kMallocActivity, /*heap_names=*/{});

  if (IsUserBuild())
    AssertNoProfileContents(packets);
  else
    AssertExpectedMallocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, ReleaseAppStartup) {
  std::string app_name = "android.perfetto.cts.app.release";
  const auto& packets =
      ProfileStartup(app_name, kMallocActivity, /*heap_names=*/{});

  if (IsUserBuild())
    AssertNoProfileContents(packets);
  else
    AssertExpectedMallocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, NonProfileableAppRuntime) {
  std::string app_name = "android.perfetto.cts.app.nonprofileable";
  const auto& packets =
      ProfileRuntime(app_name, kMallocActivity, /*heap_names=*/{});
  if (IsUserBuild())
    AssertNoProfileContents(packets);
  else
    AssertExpectedMallocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, NonProfileableAppStartup) {
  std::string app_name = "android.perfetto.cts.app.nonprofileable";
  const auto& packets =
      ProfileStartup(app_name, kMallocActivity, /*heap_names=*/{});
  if (IsUserBuild())
    AssertNoProfileContents(packets);
  else
    AssertExpectedMallocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, JavaHeapRuntime) {
  std::string app_name = "android.perfetto.cts.app.debuggable";
  const auto& packets = ProfileRuntime(app_name, kJavaAllocActivity,
                                       /*heap_names=*/{"com.android.art"});
  AssertExpectedArtAllocsPresent(packets);
  StopApp(app_name);
}

TEST(HeapprofdCtsTest, JavaHeapStartup) {
  std::string app_name = "android.perfetto.cts.app.debuggable";
  const auto& packets = ProfileStartup(app_name, kJavaAllocActivity,
                                       /*heap_names=*/{"com.android.art"});
  AssertExpectedArtAllocsPresent(packets);
  StopApp(app_name);
}

}  // namespace
}  // namespace perfetto
<|endoftext|>


<commit_before>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <sstream>

#include "zorbaerrors/dict.h"
#include "zorbaerrors/err.h"

#include "error_util.h"
#include "stl_util.h"

#ifndef WIN32
# include <cstdio>
# include <cstring>
#else
# include <windows.h>
# include <tchar.h>
# include <strsafe.h>
#endif /* WIN32 */

namespace zorba {
namespace os_error {

using namespace std;

////////// exception //////////////////////////////////////////////////////////

exception::~exception() throw() {
  // out-of-line since it's virtual
}

string exception::make_what( char const *function, char const *path,
                             char const *err_string ) {
  ostringstream oss;
  if ( path && *path )
    oss << '"' << path << "\": ";
  if ( err_string )
    oss << format_err_string( function, err_string );
  else
    oss << get_err_string( function );
  return oss.str();
}

///////////////////////////////////////////////////////////////////////////////

string format_err_string( char const *function, char const *err_string ) {
  if ( function && *function ) {
    using namespace internal::err;
    parameters::value_type result =
      err::dict::lookup( ZED( FunctionFailed_12o ) );
    parameters const params( ERROR_PARAMS( function, err_string ) );
    params.substitute( &result );
    return result;
  } else {
    return err_string;
  }
}

string format_err_string( char const *function, code_type code,
                          char const *err_string ) {
  internal::err::parameters params;
  internal::err::parameters::value_type result;
  if ( function && *function ) {
    result = err::dict::lookup( ZED( FunctionFailedErrorCodeMessage_123 ) );
    params = ERROR_PARAMS( function, code, err_string );
  } else {
    result = err::dict::lookup( ZED( ErrorCodeMessage_12 ) );
    params = ERROR_PARAMS( code, err_string );
  }
  params.substitute( &result );
  return result;
}

string get_err_string( char const *function, code_type code ) {
#ifndef WIN32
  char err_string[ 128 ];
  ::strerror_r( code, err_string, sizeof( err_string ) );
#else
  LPWSTR werr_string;
  FormatMessage(
    FORMAT_MESSAGE_ALLOCATE_BUFFER
    | FORMAT_MESSAGE_FROM_SYSTEM
    | FORMAT_MESSAGE_IGNORE_INSERTS,
    NULL, code, MAKELANGID( LANG_NEUTRAL, SUBLANG_DEFAULT ),
    reinterpret_cast<LPWSTR>( &werr_string ), 0, NULL
  );
  int const err_size = ::wcslen( werr_string ) * 3;
  ztd::auto_vec<char> const err_buf( new char[ err_size ] );
  char *const err_string = err_buf.get();
  WideCharToMultiByte(
    CP_UTF8, 0, werr_string, -1, err_string, err_size, NULL, NULL
  );
  LocalFree( werr_string );
#endif /* WIN32 */
  return format_err_string( function, code, err_string );
}

///////////////////////////////////////////////////////////////////////////////

} // namespace os_error
} // namespace zorba
/*
 * Local variables:
 * mode: c++
 * End:
 */
/* vim:set et sw=2 ts=2: */
<commit_msg>Fix for a Windows broken build in error_util<commit_after>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <sstream>

# include <cstring>
#ifndef WIN32
# include <cstdio>
#else
# include <windows.h>
#endif /* WIN32 */

#include "zorbaerrors/dict.h"
#include "zorbaerrors/err.h"

#include "error_util.h"
#include "stl_util.h"

namespace zorba {
namespace os_error {

using namespace std;

////////// exception //////////////////////////////////////////////////////////

exception::~exception() throw() {
  // out-of-line since it's virtual
}

string exception::make_what( char const *function, char const *path,
                             char const *err_string ) {
  ostringstream oss;
  if ( path && *path )
    oss << '"' << path << "\": ";
  if ( err_string )
    oss << format_err_string( function, err_string );
  else
    oss << get_err_string( function );
  return oss.str();
}

///////////////////////////////////////////////////////////////////////////////

string format_err_string( char const *function, char const *err_string ) {
  if ( function && *function ) {
    using namespace internal::err;
    parameters::value_type result =
      err::dict::lookup( ZED( FunctionFailed_12o ) );
    parameters const params( ERROR_PARAMS( function, err_string ) );
    params.substitute( &result );
    return result;
  } else {
    return err_string;
  }
}

string format_err_string( char const *function, code_type code,
                          char const *err_string ) {
  internal::err::parameters params;
  internal::err::parameters::value_type result;
  if ( function && *function ) {
    result = err::dict::lookup( ZED( FunctionFailedErrorCodeMessage_123 ) );
    params = ERROR_PARAMS( function, code, err_string );
  } else {
    result = err::dict::lookup( ZED( ErrorCodeMessage_12 ) );
    params = ERROR_PARAMS( code, err_string );
  }
  params.substitute( &result );
  return result;
}

string get_err_string( char const *function, code_type code ) {
#ifndef WIN32
  char err_string[ 128 ];
  ::strerror_r( code, err_string, sizeof( err_string ) );
#else
  LPWSTR werr_string;
  FormatMessage(
    FORMAT_MESSAGE_ALLOCATE_BUFFER
    | FORMAT_MESSAGE_FROM_SYSTEM
    | FORMAT_MESSAGE_IGNORE_INSERTS,
    NULL, code, MAKELANGID( LANG_NEUTRAL, SUBLANG_DEFAULT ),
    reinterpret_cast<LPWSTR>( &werr_string ), 0, NULL
  );
  int const err_size = ::wcslen( werr_string ) * 3;
  ztd::auto_vec<char> const err_buf( new char[ err_size ] );
  char *const err_string = err_buf.get();
  WideCharToMultiByte(
    CP_UTF8, 0, werr_string, -1, err_string, err_size, NULL, NULL
  );
  LocalFree( werr_string );
#endif /* WIN32 */
  return format_err_string( function, code, err_string );
}

///////////////////////////////////////////////////////////////////////////////

} // namespace os_error
} // namespace zorba
/*
 * Local variables:
 * mode: c++
 * End:
 */
/* vim:set et sw=2 ts=2: */
<|endoftext|>