Agora Server Gateway SDK C++ API Reference
AgoraMediaBase.h
Go to the documentation of this file.
1//
2// Agora Engine SDK
3//
4// Created by Sting Feng in 2017-11.
5// Copyright (c) 2017 Agora.io. All rights reserved.
6
7#pragma once // NOLINT(build/header_guard)
8
9#include <cstring>
10#include <stdint.h>
11#include <limits>
12#include <stddef.h>
13
14#ifndef OPTIONAL_ENUM_SIZE_T
15#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)
16#define OPTIONAL_ENUM_SIZE_T enum : size_t
17#else
18#define OPTIONAL_ENUM_SIZE_T enum
19#endif
20#endif
21
22namespace agora {
23namespace rtc {
24
25typedef unsigned int uid_t;
26typedef unsigned int track_id_t;
27typedef unsigned int conn_id_t;
28
29static const unsigned int DEFAULT_CONNECTION_ID = 0;
30static const unsigned int DUMMY_CONNECTION_ID = (std::numeric_limits<unsigned int>::max)();
31
32
34
43};
44
47 size_t channels;
49
51 : sample_rate(0),
52 channels(0),
54};
55
62
68};
69
70} // namespace rtc
71
73namespace media {
77enum MEDIA_SOURCE_TYPE {
81 AUDIO_PLAYOUT_SOURCE = 0,
85 AUDIO_RECORDING_SOURCE = 1,
89 PRIMARY_CAMERA_SOURCE = 2,
93 SECONDARY_CAMERA_SOURCE = 3,
97 PRIMARY_SCREEN_SOURCE = 4,
101 SECONDARY_SCREEN_SOURCE = 5,
105 CUSTOM_VIDEO_SOURCE = 6,
109 MEDIA_PLAYER_SOURCE = 7,
113 RTC_IMAGE_PNG_SOURCE = 8,
117 RTC_IMAGE_JPEG_SOURCE = 9,
121 RTC_IMAGE_GIF_SOURCE = 10,
125 REMOTE_VIDEO_SOURCE = 11,
129 TRANSCODED_VIDEO_SOURCE = 12,
133 UNKNOWN_MEDIA_SOURCE = 100
134};
135namespace base {
136
137typedef void* view_t;
138
139typedef const char* user_id_t;
140
141static const uint8_t kMaxCodecNameLength = 50;
143
145
148enum MAX_METADATA_SIZE_TYPE {
149 MAX_METADATA_SIZE_IN_BYTE = 1024
150};
151
156struct PacketOptions {
160 uint32_t timestamp;
161 // Audio level indication.
162 uint8_t audioLevelIndication;
163 PacketOptions()
164 : timestamp(0),
165 audioLevelIndication(127) {}
166};
168
177 // Stereo, 32 kHz, 60 ms (2 * 32 * 60)
184 kMaxDataSizeSamples = 3840,
186 kMaxDataSizeBytes = kMaxDataSizeSamples * sizeof(int16_t),
187 };
188
205 int16_t data_[kMaxDataSizeSamples];
206
208 if(this == &src) {
209 return *this;
210 }
211
212 this->capture_timestamp = src.capture_timestamp;
213 this->samples_per_channel_ = src.samples_per_channel_;
214 this->sample_rate_hz_ = src.sample_rate_hz_;
215 this->bytes_per_sample = src.bytes_per_sample;
216 this->num_channels_ = src.num_channels_;
217
218 size_t length = src.samples_per_channel_ * src.num_channels_;
219 if (length > kMaxDataSizeSamples) {
220 length = kMaxDataSizeSamples;
221 }
222
223 memcpy(this->data_, src.data_, length * sizeof(int16_t));
224
225 return *this;
226 }
227
229 : capture_timestamp(0),
230 samples_per_channel_(0),
231 sample_rate_hz_(0),
232 num_channels_(0),
233 bytes_per_sample(rtc::TWO_BYTES_PER_SAMPLE) {
234 memset(data_, 0, sizeof(data_));
235 }
236
238 : capture_timestamp(src.capture_timestamp),
239 samples_per_channel_(src.samples_per_channel_),
240 sample_rate_hz_(src.sample_rate_hz_),
241 num_channels_(src.num_channels_),
242 bytes_per_sample(src.bytes_per_sample) {
243 size_t length = src.samples_per_channel_ * src.num_channels_;
244 if (length > kMaxDataSizeSamples) {
245 length = kMaxDataSizeSamples;
246 }
247
248 memcpy(this->data_, src.data_, length * sizeof(int16_t));
249 }
250};
251
253
255enum AUDIO_DUAL_MONO_MODE {
257 AUDIO_DUAL_MONO_STEREO = 0,
259 AUDIO_DUAL_MONO_L = 1,
261 AUDIO_DUAL_MONO_R = 2,
263 AUDIO_DUAL_MONO_MIX = 3
264};
266
269 public:
278 virtual void onFrame(AudioPcmFrame* frame) = 0;
279 virtual ~IAudioFrameObserver() {}
280};
282
323};
324
326
329enum RENDER_MODE_TYPE {
334 RENDER_MODE_HIDDEN = 1,
340 RENDER_MODE_FIT = 2,
345 RENDER_MODE_ADAPTIVE = 3,
346};
348
354 : type(VIDEO_BUFFER_RAW_DATA),
355 format(VIDEO_PIXEL_UNKNOWN),
356 buffer(NULL),
357 stride(0),
358 height(0),
359 cropLeft(0),
360 cropTop(0),
361 cropRight(0),
362 cropBottom(0),
363 rotation(0),
364 timestamp(0),
365 eglContext(NULL),
366 eglType(EGL_CONTEXT10),
367 textureId(0),
368 metadata_buffer(NULL),
369 metadata_size(0),
370 alphaBuffer(NULL){}
371
373
376 enum EGL_CONTEXT_TYPE {
380 EGL_CONTEXT10 = 0,
384 EGL_CONTEXT14 = 1,
385 };
387
394 VIDEO_BUFFER_RAW_DATA = 1,
398 VIDEO_BUFFER_ARRAY = 2,
402 VIDEO_BUFFER_TEXTURE = 3,
403 };
404
416 void* buffer;
455 long long timestamp;
457
462 void *eglContext;
466 EGL_CONTEXT_TYPE eglType;
470 int textureId;
474 float matrix[16];
479 uint8_t* metadata_buffer;
484 int metadata_size;
486
498 uint8_t* alphaBuffer;
499};
500
507 width(0),
508 height(0),
509 yStride(0),
510 uStride(0),
511 vStride(0),
512 yBuffer(NULL),
513 uBuffer(NULL),
514 vBuffer(NULL),
515 rotation(0),
516 renderTimeMs(0),
517 avsync_type(0),
518 metadata_buffer(NULL),
519 metadata_size(0),
520 sharedContext(0),
521 textureId(0),
522 alphaBuffer(NULL){}
523
531 int width;
551 uint8_t* yBuffer;
555 uint8_t* uBuffer;
559 uint8_t* vBuffer;
576
580 uint8_t* metadata_buffer;
585 int metadata_size;
589 void* sharedContext;
593 int textureId;
597 float matrix[16];
603 uint8_t* alphaBuffer;
605};
606
609 public:
617 virtual void onFrame(const VideoFrame* frame) = 0;
618 virtual ~IVideoFrameObserver() {}
619 virtual bool isExternal() { return true; }
620 virtual VIDEO_PIXEL_FORMAT getVideoPixelFormatPreference() { return VIDEO_PIXEL_UNKNOWN; }
621};
622
623enum MEDIA_PLAYER_SOURCE_TYPE {
629 MEDIA_PLAYER_SOURCE_DEFAULT,
635 MEDIA_PLAYER_SOURCE_FULL_FEATURED,
640 MEDIA_PLAYER_SOURCE_SIMPLE,
641};
643
649};
650
651} // namespace base
652
657 public:
665 FRAME_TYPE_PCM16 = 0,
666 };
670 struct AudioFrame {
694 void* buffer;
704
705 AudioFrame() : type(FRAME_TYPE_PCM16),
706 samplesPerChannel(0),
707 bytesPerSample(rtc::TWO_BYTES_PER_SAMPLE),
708 channels(0),
709 samplesPerSec(0),
710 buffer(NULL),
711 renderTimeMs(0),
712 avsync_type(0) {}
713 };
714
715 public:
717
726 virtual bool onRecordAudioFrame(const char* channelId, AudioFrame& audioFrame) = 0;
735 virtual bool onPlaybackAudioFrame(const char* channelId, AudioFrame& audioFrame) = 0;
744 virtual bool onMixedAudioFrame(const char* channelId, AudioFrame& audioFrame) = 0;
754 virtual bool onPlaybackAudioFrameBeforeMixing(const char* channelId, base::user_id_t userId, AudioFrame& audioFrame) {
755 (void) channelId;
756 (void) userId;
757 (void) audioFrame;
758 return true;
759 }
760};
761
766 public:
777 virtual bool onPlaybackAudioFrameBeforeMixing(const char* channelId, rtc::uid_t uid, AudioFrame& audioFrame) = 0;
778};
779
784 public:
795 virtual bool OnEncodedVideoFrame(rtc::uid_t uid, const uint8_t* imageBuffer, size_t length,
796 const rtc::EncodedVideoFrameInfo& videoEncodedFrameInfo) = 0;
797
799};
800
805 public:
806 typedef media::base::VideoFrame VideoFrame;
807
809 PROCESS_MODE_READ_ONLY, // Observer works as a pure renderer and will not modify the original frame.
810 PROCESS_MODE_READ_WRITE, // Observer works as a filter that will process the video frame and affect the following frame processing in SDK.
811 };
812
813 public:
815
817
835 virtual bool onCaptureVideoFrame(VideoFrame& videoFrame) = 0;
837
855 virtual bool onPreEncodeVideoFrame(VideoFrame& videoFrame) = 0;
856
858 virtual bool onSecondaryCameraCaptureVideoFrame(VideoFrame& videoFrame) = 0;
859
877 virtual bool onSecondaryPreEncodeCameraVideoFrame(VideoFrame& videoFrame) = 0;
880
895 virtual bool onScreenCaptureVideoFrame(VideoFrame& videoFrame) = 0;
896
897 virtual bool onPreEncodeScreenVideoFrame(VideoFrame& videoFrame) = 0;
898
915 virtual bool onMediaPlayerVideoFrame(VideoFrame& videoFrame, int mediaPlayerId) = 0;
916
917 virtual bool onSecondaryScreenCaptureVideoFrame(VideoFrame& videoFrame) = 0;
918
919 virtual bool onSecondaryPreEncodeScreenVideoFrame(VideoFrame& videoFrame) = 0;
937 virtual bool onRenderVideoFrame(const char* channelId, rtc::uid_t remoteUid, VideoFrame& videoFrame) = 0;
938
939 virtual bool onTranscodedVideoFrame(VideoFrame& videoFrame) = 0;
940
945 virtual VIDEO_FRAME_PROCESS_MODE getVideoFrameProcessMode() {
946 return PROCESS_MODE_READ_ONLY;
947 }
948
949 /*
950 * Occurs each time needs to get preference video frame type.
951 *
952 * @return preference video pixel format.
953 */
954 virtual base::VIDEO_PIXEL_FORMAT getVideoPixelFormatPreference() { return base::VIDEO_PIXEL_I420; }
955
963 virtual bool getRotationApplied() { return false; }
965
973 virtual bool getMirrorApplied() { return false; }
974
976
993 virtual uint32_t getObservedFramePosition() {
995 }
996
1004 virtual bool isExternal() { return true; }
1005};
1007
1009
1011#define MAX_CONTENT_INSPECT_MODULE_COUNT 32
1012enum CONTENT_INSPECT_RESULT {
1013 CONTENT_INSPECT_NEUTRAL = 1,
1014 CONTENT_INSPECT_SEXY = 2,
1015 CONTENT_INSPECT_PORN = 3,
1016};
1017enum CONTENT_INSPECT_DEVICE_TYPE{
1018 CONTENT_INSPECT_DEVICE_INVALID = 0,
1019 CONTENT_INSPECT_DEVICE_AGORA = 1,
1020 CONTENT_INSPECT_DEVICE_HIVE = 2,
1021 CONTENT_INSPECT_DEVICE_TUPU = 3
1022};
1023enum CONTENT_INSPECT_TYPE {
1027CONTENT_INSPECT_INVALIDE = 0,
1031CONTENT_INSPECT_MODERATION = 1,
1035CONTENT_INSPECT_SUPERVISE = 2
1036};
1037struct ContentInspectModule {
1041 CONTENT_INSPECT_TYPE type;
1045 unsigned int frequency;
1046};
1050enum EXTERNAL_VIDEO_SOURCE_TYPE {
1054 VIDEO_FRAME = 0,
1058 ENCODED_VIDEO_FRAME,
1059};
1061
1062} // namespace media
1063} // namespace agora
#define OPTIONAL_ENUM_SIZE_T
Definition: AgoraMediaBase.h:18
Definition: AgoraMediaBase.h:656
virtual bool onPlaybackAudioFrameBeforeMixing(const char *channelId, base::user_id_t userId, AudioFrame &audioFrame)
Definition: AgoraMediaBase.h:754
virtual ~IAudioFrameObserverBase()
Definition: AgoraMediaBase.h:716
virtual bool onRecordAudioFrame(const char *channelId, AudioFrame &audioFrame)=0
virtual bool onMixedAudioFrame(const char *channelId, AudioFrame &audioFrame)=0
AUDIO_FRAME_TYPE
Definition: AgoraMediaBase.h:661
virtual bool onPlaybackAudioFrame(const char *channelId, AudioFrame &audioFrame)=0
Definition: AgoraMediaBase.h:765
virtual bool onPlaybackAudioFrameBeforeMixing(const char *channelId, rtc::uid_t uid, AudioFrame &audioFrame)=0
Definition: AgoraMediaBase.h:783
virtual bool OnEncodedVideoFrame(rtc::uid_t uid, const uint8_t *imageBuffer, size_t length, const rtc::EncodedVideoFrameInfo &videoEncodedFrameInfo)=0
virtual ~IVideoEncodedFrameObserver()
Definition: AgoraMediaBase.h:798
Definition: AgoraMediaBase.h:804
VIDEO_FRAME_PROCESS_MODE
Definition: AgoraMediaBase.h:808
@ PROCESS_MODE_READ_WRITE
Definition: AgoraMediaBase.h:810
@ PROCESS_MODE_READ_ONLY
Definition: AgoraMediaBase.h:809
virtual bool onPreEncodeVideoFrame(VideoFrame &videoFrame)=0
virtual bool getMirrorApplied()
Definition: AgoraMediaBase.h:973
virtual ~IVideoFrameObserver()
Definition: AgoraMediaBase.h:814
media::base::VideoFrame VideoFrame
Definition: AgoraMediaBase.h:806
unsigned int track_id_t
Definition: AgoraMediaBase.h:26
BYTES_PER_SAMPLE
Definition: AgoraMediaBase.h:38
@ TWO_BYTES_PER_SAMPLE
Definition: AgoraMediaBase.h:42
unsigned int uid_t
Definition: AgoraMediaBase.h:25
unsigned int conn_id_t
Definition: AgoraMediaBase.h:27
static const unsigned int DUMMY_CONNECTION_ID
Definition: AgoraMediaBase.h:30
RAW_AUDIO_FRAME_OP_MODE_TYPE
Definition: AgoraMediaBase.h:56
@ RAW_AUDIO_FRAME_OP_MODE_READ_WRITE
Definition: AgoraMediaBase.h:67
@ RAW_AUDIO_FRAME_OP_MODE_READ_ONLY
Definition: AgoraMediaBase.h:61
static const unsigned int DEFAULT_CONNECTION_ID
Definition: AgoraMediaBase.h:29
Definition: AgoraBase.h:86
VIDEO_PIXEL_FORMAT
Definition: AgoraMediaBase.h:286
@ VIDEO_PIXEL_BGRA
Definition: AgoraMediaBase.h:298
@ VIDEO_PIXEL_UNKNOWN
Definition: AgoraMediaBase.h:290
@ VIDEO_PIXEL_NV21
Definition: AgoraMediaBase.h:302
@ VIDEO_PIXEL_I420
Definition: AgoraMediaBase.h:294
@ VIDEO_TEXTURE_2D
Definition: AgoraMediaBase.h:314
@ VIDEO_TEXTURE_OES
Definition: AgoraMediaBase.h:318
@ VIDEO_PIXEL_RGBA
Definition: AgoraMediaBase.h:306
@ VIDEO_PIXEL_NV12
Definition: AgoraMediaBase.h:310
@ VIDEO_PIXEL_I422
Definition: AgoraMediaBase.h:322
void * view_t
Definition: AgoraBase.h:1199
const char * user_id_t
Definition: AgoraBase.h:1198
VIDEO_MODULE_POSITION
Definition: AgoraMediaBase.h:644
@ POSITION_PRE_ENCODER
Definition: AgoraMediaBase.h:647
@ POSITION_POST_CAPTURER
Definition: AgoraMediaBase.h:645
@ POSITION_PRE_RENDERER
Definition: AgoraMediaBase.h:646
@ POSITION_POST_FILTERS
Definition: AgoraMediaBase.h:648
Definition: AgoraMediaBase.h:670
agora::rtc::BYTES_PER_SAMPLE bytesPerSample
Definition: AgoraMediaBase.h:682
int samplesPerChannel
Definition: AgoraMediaBase.h:678
void * buffer
Definition: AgoraMediaBase.h:694
AUDIO_FRAME_TYPE type
Definition: AgoraMediaBase.h:674
int samplesPerSec
Definition: AgoraMediaBase.h:690
int avsync_type
Definition: AgoraMediaBase.h:703
int64_t renderTimeMs
Definition: AgoraMediaBase.h:702
int channels
Definition: AgoraMediaBase.h:686
AudioFrame()
Definition: AgoraMediaBase.h:705
Definition: AgoraMediaBase.h:172
AudioPcmFrame()
Definition: AgoraMediaBase.h:228
uint32_t capture_timestamp
Definition: AgoraMediaBase.h:191
size_t num_channels_
Definition: AgoraMediaBase.h:200
int16_t data_[kMaxDataSizeSamples]
Definition: AgoraMediaBase.h:205
int sample_rate_hz_
Definition: AgoraMediaBase.h:197
AudioPcmFrame & operator=(const AudioPcmFrame &src)
Definition: AgoraMediaBase.h:207
AudioPcmFrame(const AudioPcmFrame &src)
Definition: AgoraMediaBase.h:237
rtc::BYTES_PER_SAMPLE bytes_per_sample
Definition: AgoraMediaBase.h:203
size_t samples_per_channel_
Definition: AgoraMediaBase.h:194
Definition: AgoraMediaBase.h:352
void * buffer
Definition: AgoraMediaBase.h:416
VIDEO_BUFFER_TYPE
Definition: AgoraMediaBase.h:390
uint8_t * alphaBuffer
Definition: AgoraMediaBase.h:498
ExternalVideoFrame()
Definition: AgoraMediaBase.h:353
int rotation
Definition: AgoraMediaBase.h:450
int cropLeft
Definition: AgoraMediaBase.h:430
int cropBottom
Definition: AgoraMediaBase.h:445
int height
Definition: AgoraMediaBase.h:425
int stride
Definition: AgoraMediaBase.h:421
int cropRight
Definition: AgoraMediaBase.h:440
long long timestamp
Definition: AgoraMediaBase.h:455
VIDEO_BUFFER_TYPE type
Definition: AgoraMediaBase.h:408
VIDEO_PIXEL_FORMAT format
Definition: AgoraMediaBase.h:412
int cropTop
Definition: AgoraMediaBase.h:435
Definition: AgoraMediaBase.h:504
int uStride
Definition: AgoraMediaBase.h:543
VIDEO_PIXEL_FORMAT type
Definition: AgoraMediaBase.h:527
uint8_t * yBuffer
Definition: AgoraMediaBase.h:551
int width
Definition: AgoraMediaBase.h:531
int yStride
Definition: AgoraMediaBase.h:539
uint8_t * uBuffer
Definition: AgoraMediaBase.h:555
int rotation
Definition: AgoraMediaBase.h:563
int64_t renderTimeMs
Definition: AgoraMediaBase.h:570
int vStride
Definition: AgoraMediaBase.h:547
uint8_t * vBuffer
Definition: AgoraMediaBase.h:559
VideoFrame()
Definition: AgoraMediaBase.h:505
int avsync_type
Definition: AgoraMediaBase.h:574
int height
Definition: AgoraMediaBase.h:535
Definition: AgoraMediaBase.h:45
size_t channels
Definition: AgoraMediaBase.h:47
int sample_rate
Definition: AgoraMediaBase.h:46
AudioParameters()
Definition: AgoraMediaBase.h:50
size_t frames_per_buffer
Definition: AgoraMediaBase.h:48
Definition: AgoraBase.h:1880