AgoraMediaBase.h

Go to the documentation of this file.

//
//  Agora Engine SDK
//
//  Created by Sting Feng in 2017-11.
//  Copyright (c) 2017 Agora.io. All rights reserved.
 
#pragma once  // NOLINT(build/header_guard)
 
#include <cstring>
#include <stdint.h>
#include <limits>
#include <stddef.h>
 
#ifndef OPTIONAL_ENUM_SIZE_T
#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)
#define OPTIONAL_ENUM_SIZE_T enum : size_t
#else
#define OPTIONAL_ENUM_SIZE_T enum
#endif
#endif
 
namespace agora {
namespace rtc {
 
typedef unsigned int uid_t;
typedef unsigned int track_id_t;
typedef unsigned int conn_id_t;
 
static const unsigned int DEFAULT_CONNECTION_ID = 0;
static const unsigned int DUMMY_CONNECTION_ID = (std::numeric_limits<unsigned int>::max)();
 
 
struct EncodedVideoFrameInfo;
 
enum BYTES_PER_SAMPLE {
  TWO_BYTES_PER_SAMPLE = 2,
};
 
struct AudioParameters {
  int sample_rate;
  size_t channels;
  size_t frames_per_buffer;
 
  AudioParameters()
      : sample_rate(0),
        channels(0),
        frames_per_buffer(0) {}
};
 
enum RAW_AUDIO_FRAME_OP_MODE_TYPE {
  RAW_AUDIO_FRAME_OP_MODE_READ_ONLY = 0,
 
  RAW_AUDIO_FRAME_OP_MODE_READ_WRITE = 2,
};
 
enum VIDEO_SOURCE_TYPE {
  VIDEO_SOURCE_CAMERA_PRIMARY = 0,
  VIDEO_SOURCE_CAMERA = VIDEO_SOURCE_CAMERA_PRIMARY,
  VIDEO_SOURCE_CAMERA_SECONDARY = 1,
  VIDEO_SOURCE_SCREEN_PRIMARY = 2,
  VIDEO_SOURCE_SCREEN = VIDEO_SOURCE_SCREEN_PRIMARY,
  VIDEO_SOURCE_SCREEN_SECONDARY = 3,
  VIDEO_SOURCE_CUSTOM = 4,
  VIDEO_SOURCE_MEDIA_PLAYER = 5,
  VIDEO_SOURCE_RTC_IMAGE_PNG = 6,
  VIDEO_SOURCE_RTC_IMAGE_JPEG = 7,
  VIDEO_SOURCE_RTC_IMAGE_GIF = 8,
  VIDEO_SOURCE_REMOTE = 9,
  VIDEO_SOURCE_TRANSCODED = 10,
 
  VIDEO_SOURCE_CAMERA_THIRD = 11,
  VIDEO_SOURCE_CAMERA_FOURTH = 12,
  VIDEO_SOURCE_SCREEN_THIRD = 13,
  VIDEO_SOURCE_SCREEN_FOURTH = 14,
  VIDEO_SOURCE_SPEECH_DRIVEN = 15,
 
  VIDEO_SOURCE_UNKNOWN = 100
};
 
class IMetadataObserver {
 public:
  virtual ~IMetadataObserver() {}
  enum METADATA_TYPE {
    UNKNOWN_METADATA = -1,
    VIDEO_METADATA = 0,
  };
  enum MAX_METADATA_SIZE_TYPE {
    INVALID_METADATA_SIZE_IN_BYTE = -1,
    DEFAULT_METADATA_SIZE_IN_BYTE = 512,
    MAX_METADATA_SIZE_IN_BYTE = 1024
  };
  struct Metadata {
    const char* channelId;
    unsigned int uid;
    unsigned int size;
    unsigned char* buffer;
    long long timeStampMs;
    Metadata() : channelId(NULL), uid(0), size(0), buffer(NULL), timeStampMs(0) {}
  };
  virtual int getMaxMetadataSize() { return DEFAULT_METADATA_SIZE_IN_BYTE; }
  virtual bool onReadyToSendMetadata(Metadata& metadata, VIDEO_SOURCE_TYPE source_type) = 0;
  virtual void onMetadataReceived(const Metadata& metadata) = 0;
};
 
}  // namespace rtc
 
namespace media {
enum MEDIA_SOURCE_TYPE {
  AUDIO_PLAYOUT_SOURCE = 0,
  AUDIO_RECORDING_SOURCE = 1,
  PRIMARY_CAMERA_SOURCE = 2,
  SECONDARY_CAMERA_SOURCE = 3,
  PRIMARY_SCREEN_SOURCE = 4,
  SECONDARY_SCREEN_SOURCE = 5,
  CUSTOM_VIDEO_SOURCE = 6,
  MEDIA_PLAYER_SOURCE = 7,
  RTC_IMAGE_PNG_SOURCE = 8,
  RTC_IMAGE_JPEG_SOURCE = 9,
  RTC_IMAGE_GIF_SOURCE = 10,
  REMOTE_VIDEO_SOURCE = 11,
  TRANSCODED_VIDEO_SOURCE = 12,
  UNKNOWN_MEDIA_SOURCE = 100
};
namespace base {
 
typedef void* view_t;
 
typedef const char* user_id_t;
 
static const uint8_t kMaxCodecNameLength = 50;
 
 
enum MAX_METADATA_SIZE_TYPE {
  MAX_METADATA_SIZE_IN_BYTE = 1024
};
 
struct PacketOptions {
  uint32_t timestamp;
  // Audio level indication.
  uint8_t audioLevelIndication;
  PacketOptions()
      : timestamp(0),
        audioLevelIndication(127) {}
};
 
struct AudioPcmFrame {
  OPTIONAL_ENUM_SIZE_T {
    // Stereo, 32 kHz, 60 ms (2 * 32 * 60)
    kMaxDataSizeSamples = 3840,
    kMaxDataSizeBytes = kMaxDataSizeSamples * sizeof(int16_t),
  };
 
  uint32_t capture_timestamp;
  size_t samples_per_channel_;
  int sample_rate_hz_;
  size_t num_channels_;
  rtc::BYTES_PER_SAMPLE bytes_per_sample;
  int16_t data_[kMaxDataSizeSamples];
 
  AudioPcmFrame& operator=(const AudioPcmFrame& src) {
    if(this == &src) {
      return *this;
    }
 
    this->capture_timestamp = src.capture_timestamp;
    this->samples_per_channel_ = src.samples_per_channel_;
    this->sample_rate_hz_ = src.sample_rate_hz_;
    this->bytes_per_sample = src.bytes_per_sample;
    this->num_channels_ = src.num_channels_;
 
    size_t length = src.samples_per_channel_ * src.num_channels_;
    if (length > kMaxDataSizeSamples) {
      length = kMaxDataSizeSamples;
    }
 
    memcpy(this->data_, src.data_, length * sizeof(int16_t));
 
    return *this;
  }
 
  AudioPcmFrame()
      : capture_timestamp(0),
        samples_per_channel_(0),
        sample_rate_hz_(0),
        num_channels_(0),
        bytes_per_sample(rtc::TWO_BYTES_PER_SAMPLE) {
    memset(data_, 0, sizeof(data_));
  }
 
  AudioPcmFrame(const AudioPcmFrame& src)
      : capture_timestamp(src.capture_timestamp),
        samples_per_channel_(src.samples_per_channel_),
        sample_rate_hz_(src.sample_rate_hz_),
        num_channels_(src.num_channels_),
        bytes_per_sample(src.bytes_per_sample) {
    size_t length = src.samples_per_channel_ * src.num_channels_;
    if (length > kMaxDataSizeSamples) {
      length = kMaxDataSizeSamples;
    }
 
    memcpy(this->data_, src.data_, length * sizeof(int16_t));
  }
};
 
 
enum AUDIO_DUAL_MONO_MODE {
  AUDIO_DUAL_MONO_STEREO = 0,
  AUDIO_DUAL_MONO_L = 1,
  AUDIO_DUAL_MONO_R = 2,
  AUDIO_DUAL_MONO_MIX = 3
};
 
class IAudioFrameObserver {
 public:
  virtual void onFrame(AudioPcmFrame* frame) = 0;
  virtual ~IAudioFrameObserver() {}
};
 
enum VIDEO_PIXEL_FORMAT {
  VIDEO_PIXEL_UNKNOWN = 0,
  VIDEO_PIXEL_I420 = 1,
  VIDEO_PIXEL_BGRA = 2,
  VIDEO_PIXEL_NV21 = 3,
  VIDEO_PIXEL_RGBA = 4,
  VIDEO_PIXEL_NV12 = 8,
  VIDEO_TEXTURE_2D = 10,
  VIDEO_TEXTURE_OES = 11,
  VIDEO_PIXEL_I422 = 16,
};
 
 
enum RENDER_MODE_TYPE {
  RENDER_MODE_HIDDEN = 1,
  RENDER_MODE_FIT = 2,
  RENDER_MODE_ADAPTIVE = 3,
};
 
struct ExternalVideoFrame {
  ExternalVideoFrame()
      : type(VIDEO_BUFFER_RAW_DATA),
        format(VIDEO_PIXEL_UNKNOWN),
        buffer(NULL),
        stride(0),
        height(0),
        cropLeft(0),
        cropTop(0),
        cropRight(0),
        cropBottom(0),
        rotation(0),
        timestamp(0),
        eglContext(NULL),
        eglType(EGL_CONTEXT10),
        textureId(0),
        metadata_buffer(NULL),
        metadata_size(0),
        alphaBuffer(NULL){}
 
 
  enum EGL_CONTEXT_TYPE {
    EGL_CONTEXT10 = 0,
    EGL_CONTEXT14 = 1,
  };
 
  enum VIDEO_BUFFER_TYPE {
    VIDEO_BUFFER_RAW_DATA = 1,
    VIDEO_BUFFER_ARRAY = 2,
    VIDEO_BUFFER_TEXTURE = 3,
  };
 
  VIDEO_BUFFER_TYPE type;
  VIDEO_PIXEL_FORMAT format;
  void* buffer;
  int stride;
  int height;
  int cropLeft;
  int cropTop;
  int cropRight;
  int cropBottom;
  int rotation;
  long long timestamp;
 
  void *eglContext;
  EGL_CONTEXT_TYPE eglType;
  int textureId;
  float matrix[16];
  uint8_t* metadata_buffer;
  int metadata_size;
 
  uint8_t* alphaBuffer;
};
 
struct VideoFrame {
  VideoFrame():
  type(VIDEO_PIXEL_UNKNOWN),
  width(0),
  height(0),
  yStride(0),
  uStride(0),
  vStride(0),
  yBuffer(NULL),
  uBuffer(NULL),
  vBuffer(NULL),
  rotation(0),
  renderTimeMs(0),
  avsync_type(0),
  metadata_buffer(NULL),
  metadata_size(0),
  sharedContext(0),
  textureId(0),
  alphaBuffer(NULL){}
 
  VIDEO_PIXEL_FORMAT type;
  int width;
  int height;
  int yStride;
  int uStride;
  int vStride;
  uint8_t* yBuffer;
  uint8_t* uBuffer;
  uint8_t* vBuffer;
  int rotation;
  int64_t renderTimeMs;
  int avsync_type;
 
  uint8_t* metadata_buffer;
  int metadata_size;
  void* sharedContext;
  int textureId;
  float matrix[16];
  uint8_t* alphaBuffer;
};
 
class IVideoFrameObserver {
 public:
  virtual void onFrame(const VideoFrame* frame) = 0;
  virtual ~IVideoFrameObserver() {}
  virtual bool isExternal() { return true; }
  virtual VIDEO_PIXEL_FORMAT getVideoPixelFormatPreference() { return VIDEO_PIXEL_UNKNOWN; }
};
 
enum MEDIA_PLAYER_SOURCE_TYPE {
  MEDIA_PLAYER_SOURCE_DEFAULT,
  MEDIA_PLAYER_SOURCE_FULL_FEATURED,
  MEDIA_PLAYER_SOURCE_SIMPLE,
};
 
enum VIDEO_MODULE_POSITION {
  POSITION_POST_CAPTURER = 1 << 0,
  POSITION_PRE_RENDERER = 1 << 1,
  POSITION_PRE_ENCODER = 1 << 2,
  POSITION_POST_FILTERS = 1 << 3,
};
 
}  // namespace base
 
class IAudioFrameObserverBase {
 public:
  enum AUDIO_FRAME_TYPE {
    FRAME_TYPE_PCM16 = 0,
  };
  struct AudioFrame {
    AUDIO_FRAME_TYPE type;
    int samplesPerChannel;
    agora::rtc::BYTES_PER_SAMPLE bytesPerSample;
    int channels;
    int samplesPerSec;
    void* buffer;
    int64_t renderTimeMs;
    int avsync_type;
 
    AudioFrame() : type(FRAME_TYPE_PCM16),
                   samplesPerChannel(0),
                   bytesPerSample(rtc::TWO_BYTES_PER_SAMPLE),
                   channels(0),
                   samplesPerSec(0),
                   buffer(NULL),
                   renderTimeMs(0),
                   avsync_type(0) {}
  };
 
 public:
  virtual ~IAudioFrameObserverBase() {}
 
  virtual bool onRecordAudioFrame(const char* channelId, AudioFrame& audioFrame) = 0;
  virtual bool onPlaybackAudioFrame(const char* channelId, AudioFrame& audioFrame) = 0;
  virtual bool onMixedAudioFrame(const char* channelId, AudioFrame& audioFrame) = 0;
  virtual bool onPlaybackAudioFrameBeforeMixing(const char* channelId, base::user_id_t userId, AudioFrame& audioFrame) {
    (void) channelId;
    (void) userId;
    (void) audioFrame;
    return true;
  }
};
 
class IAudioFrameObserver : public IAudioFrameObserverBase {
 public:
  using IAudioFrameObserverBase::onPlaybackAudioFrameBeforeMixing;
  virtual bool onPlaybackAudioFrameBeforeMixing(const char* channelId, rtc::uid_t uid, AudioFrame& audioFrame) = 0;
};
 
class IVideoEncodedFrameObserver {
 public:
  virtual bool OnEncodedVideoFrame(rtc::uid_t uid, const uint8_t* imageBuffer, size_t length,
                                   const rtc::EncodedVideoFrameInfo& videoEncodedFrameInfo) = 0;
 
  virtual ~IVideoEncodedFrameObserver() {}
};
 
class IVideoFrameObserver {
 public:
  typedef media::base::VideoFrame VideoFrame;
 
  enum VIDEO_FRAME_PROCESS_MODE {
    PROCESS_MODE_READ_ONLY, // Observer works as a pure renderer and will not modify the original frame.
    PROCESS_MODE_READ_WRITE, // Observer works as a filter that will process the video frame and affect the following frame processing in SDK.
  };
 
 public:
  virtual ~IVideoFrameObserver() {}
 
 
  virtual bool onCaptureVideoFrame(VideoFrame& videoFrame) = 0;
 
  virtual bool onPreEncodeVideoFrame(VideoFrame& videoFrame) = 0;
 
  virtual bool onSecondaryCameraCaptureVideoFrame(VideoFrame& videoFrame) = 0;
 
  virtual bool onSecondaryPreEncodeCameraVideoFrame(VideoFrame& videoFrame) = 0;
 
  virtual bool onScreenCaptureVideoFrame(VideoFrame& videoFrame) = 0;
 
  virtual bool onPreEncodeScreenVideoFrame(VideoFrame& videoFrame) = 0;
 
  virtual bool onMediaPlayerVideoFrame(VideoFrame& videoFrame, int mediaPlayerId) = 0;
 
  virtual bool onSecondaryScreenCaptureVideoFrame(VideoFrame& videoFrame) = 0;
 
  virtual bool onSecondaryPreEncodeScreenVideoFrame(VideoFrame& videoFrame) = 0;
  virtual bool onRenderVideoFrame(const char* channelId, rtc::uid_t remoteUid, VideoFrame& videoFrame) = 0;
 
  virtual bool onTranscodedVideoFrame(VideoFrame& videoFrame) = 0;
 
  virtual VIDEO_FRAME_PROCESS_MODE getVideoFrameProcessMode() {
    return PROCESS_MODE_READ_ONLY;
  }
 
  /*
   * Occurs each time needs to get preference video frame type.
   *
   * @return preference video pixel format.
   */
  virtual base::VIDEO_PIXEL_FORMAT getVideoPixelFormatPreference() { return base::VIDEO_PIXEL_I420; }
 
  virtual bool getRotationApplied() { return false; }
 
  virtual bool getMirrorApplied() { return false; }
 
 
  virtual uint32_t getObservedFramePosition() {
    return base::POSITION_POST_CAPTURER | base::POSITION_PRE_RENDERER;
  }
 
  virtual bool isExternal() { return true; }
};
 
 
#define MAX_CONTENT_INSPECT_MODULE_COUNT 32
enum CONTENT_INSPECT_RESULT {
  CONTENT_INSPECT_NEUTRAL = 1,
  CONTENT_INSPECT_SEXY = 2,
  CONTENT_INSPECT_PORN = 3,
};
enum CONTENT_INSPECT_DEVICE_TYPE{
    CONTENT_INSPECT_DEVICE_INVALID = 0,
    CONTENT_INSPECT_DEVICE_AGORA = 1,
    CONTENT_INSPECT_DEVICE_HIVE = 2,
    CONTENT_INSPECT_DEVICE_TUPU = 3
};
enum CONTENT_INSPECT_TYPE {
CONTENT_INSPECT_INVALIDE = 0,
CONTENT_INSPECT_MODERATION = 1,
CONTENT_INSPECT_SUPERVISE = 2
};
struct ContentInspectModule {
  CONTENT_INSPECT_TYPE type;
  unsigned int frequency;
};
enum EXTERNAL_VIDEO_SOURCE_TYPE {
  VIDEO_FRAME = 0,
  ENCODED_VIDEO_FRAME,
};
 
}  // namespace media
}  // namespace agora