Administrator/NANOKVM-MaixCDK-MIRROR
1
0
Fork 1
mirror of https://github.com/sipeed/MaixCDK.git synced 2026-09-10 21:59:54 -05:00
Code Issues Packages Projects Releases 1 Wiki Activity

add VBR encode support

Browse Source
This commit is contained in:
916BGAI
2025-12-18 16:05:39 +08:00
parent b881c0a0a5
commit 1a33050ed7
10 changed files with 365 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
components/3rd_party/datachannel/CMakeLists.txt vendored
View File

@@ -19,16 +19,18 @@ if (compile_from_src)
aux_source_directory("${srcs_path}/src/impl" ADD_SRCS)
list(APPEND ADD_REQUIREMENTS openssl json libjuice libsrtp plog usrsctp cpp-httplib)
list(APPEND ADD_DEFINITIONS_PRIVATE -DJUICE_STATIC -DRTC_ENABLE_MEDIA=1 -DRTC_ENABLE_WEBSOCKET=1 -DRTC_EXPORTS -DRTC_SYSTEM_JUICE=0 -DRTC_SYSTEM_SRTP=0 -DUSE_GNUTLS=0 -DUSE_NICE=0 -D_GNU_SOURCE -Ddatachannel_EXPORTS -DNDEBUG -w)
register_component(DYNAMIC)
else()
list(APPEND ADD_REQUIREMENTS json cpp-httplib)
if (PLATFORM_MAIXCAM)
list(APPEND ADD_DYNAMIC_LIB "so/maixcam/libdatachannel.so")
list(APPEND ADD_DYNAMIC_LIB "lib/maixcam/libdatachannel.so")
elseif(PLATFORM_MAIXCAM2)
list(APPEND ADD_DYNAMIC_LIB "so/maixcam2/libdatachannel.so")
list(APPEND ADD_DYNAMIC_LIB "lib/maixcam2/libdatachannel.so")
else()
list(APPEND ADD_REQUIREMENTS libdatachannel)
endif()
register_component()
endif()
register_component()

9
components/maixcam_lib/include/maixcam/sophgo_middleware.hpp
View File

@@ -41,6 +41,14 @@ typedef struct {
int count;
} mmf_h265_stream_t;
typedef enum
{
MMF_VENC_RCMODE_CBR = 0,
MMF_VENC_RCMODE_VBR,
MMF_VENC_RCMODE_FIXQP,
MMF_VENC_RCMODE_MAX,
} mmf_venc_rc_mode_e;
typedef struct {
uint8_t type; // 0, jpg; 1, h265; 2, h264
int w;
@@ -51,6 +59,7 @@ typedef struct {
int intput_fps; // h264/h265
int output_fps; // h264/h265
int bitrate; // h264/h265
mmf_venc_rc_mode_e rc_mode; // h264/h265
} mmf_venc_cfg_t;
typedef struct {

9
components/maixcam_lib/include/maixcam2/ax_middleware.hpp
View File

@@ -59,6 +59,14 @@ namespace maix::middleware::maixcam2 {
AX_VENC_TYPE_MJPG,
} ax_venc_type_e;
typedef enum
{
AX_VENC_RCMODE_CBR = 0,
AX_VENC_RCMODE_VBR,
AX_VENC_RCMODE_FIXQP,
AX_VENC_RCMODE_MAX,
} ax_venc_rc_mode_e;
typedef enum {
AX_VDEC_TYPE_JPG = 0,
AX_VDEC_TYPE_H264,
@@ -89,6 +97,7 @@ namespace maix::middleware::maixcam2 {
typedef struct {
bool en;
ax_venc_type_e type;
ax_venc_rc_mode_e rc_mode;
int w;
int h;
AX_IMG_FORMAT_E fmt;

9
components/vision/include/maix_video.hpp
View File

@@ -31,16 +31,15 @@ namespace maix::video
VIDEO_ENC_MP4_CBR, // Deprecated
VIDEO_DEC_H265_CBR, // Deprecated
VIDEO_DEC_MP4_CBR, // Deprecated
VIDEO_H264_CBR, // Deprecated
VIDEO_H265_CBR, // Deprecated
VIDEO_H264_CBR_MP4, // Deprecated
VIDEO_H265_CBR_MP4, // Deprecated
VIDEO_H264,
VIDEO_H264_MP4,
VIDEO_H264_FLV,
VIDEO_H264_CBR,
VIDEO_H264_VBR,
VIDEO_H265,
VIDEO_H265_MP4,
VIDEO_H265_CBR,
VIDEO_H265_VBR,
};
/**

21
components/vision/include/maix_webrtc.hpp
View File

@@ -28,6 +28,17 @@ namespace maix::webrtc
WEBRTC_STREAM_H265,
};
/**
* The rc type of webrtc
* @maixpy maix.webrtc.WebRTCRCType
*/
enum class WebRTCRCType
{
WEBRTC_RC_NONE = 0, // format invalid
WEBRTC_RC_CBR,
WEBRTC_RC_VBR,
};
/**
* Region class
* @maixpy maix.webrtc.Region
@@ -86,18 +97,20 @@ namespace maix::webrtc
* @brief Construct a new WebRTC object
* @param ip listen ip
* @param port listen port
* @param fps video fps
* @param stream_type stream type
* @param rc_type rc type
* @param bitrate video bitrate
* @param gop video gop
* @param signaling_ip signaling server bind ip
* @param signaling_port signaling server bind port
* @param http_server whether to enable the HTTP server
* @param stun_server STUN server address
* @param http_server whether to enable the HTTP server
* @maixpy maix.webrtc.WebRTC.__init__
* @maixcdk maix.webrtc.WebRTC.WebRTC
*/
WebRTC(std::string ip = std::string(), int port = 8000, int fps = 60,
WebRTC(std::string ip = std::string(), int port = 8000,
webrtc::WebRTCStreamType stream_type = webrtc::WebRTCStreamType::WEBRTC_STREAM_H264,
webrtc::WebRTCRCType rc_type = webrtc::WebRTCRCType::WEBRTC_RC_CBR,
int bitrate = 3000 * 1000, int gop = 60, std::string signaling_ip = std::string(),
int signaling_port = 8001, const std::string &stun_server = "stun:stun.l.google.com:19302",
bool http_server = true);
@@ -250,9 +263,9 @@ namespace maix::webrtc
int _port;
std::string _signaling_ip;
int _signaling_port;
int _fps;
int _gop;
webrtc::WebRTCStreamType _stream_type;
webrtc::WebRTCRCType _rc_type;
bool _is_start;
thread::Thread *_video_thread;
std::string _stun_server;

10
components/vision/port/linux/maix_webrtc_linux.cpp
View File

@@ -34,17 +34,15 @@ namespace maix::webrtc
return err::ERR_NOT_IMPL;
}
WebRTC::WebRTC(std::string ip, int port, int fps,
webrtc::WebRTCStreamType stream_type, int bitrate, int gop,
std::string signaling_ip, int signaling_port,
const std::string &stun_server,
bool http_server)
WebRTC::WebRTC(std::string ip, int port, webrtc::WebRTCStreamType stream_type,
webrtc::WebRTCRCType rc_type, int bitrate, int gop, std::string signaling_ip,
int signaling_port, const std::string &stun_server, bool http_server)
{
(void)ip;
(void)port;
(void)fps;
(void)gop;
(void)stream_type;
(void)rc_type;
(void)bitrate;
(void)signaling_ip;
(void)signaling_port;

229
components/vision/port/maixcam/maix_video_mmf.cpp
View File

@@ -56,19 +56,46 @@ namespace maix::video
}
if (!strcmp(suffix, ".h264")) {
video_type = video::VIDEO_H264;
switch (type) {
case video::VIDEO_H264: // fall through
case video::VIDEO_H264_CBR:
video_type = video::VIDEO_H264_CBR;
break;
case video::VIDEO_H264_VBR:
video_type = video::VIDEO_H264_VBR;
break;
default:
err::check_raise(err::ERR_RUNTIME, "Unsupported video type!");
break;
}
} else if (!strcmp(suffix, ".h265")) {
video_type = video::VIDEO_H264;
switch (type) {
case video::VIDEO_H265: // fall through
case video::VIDEO_H265_CBR:
video_type = video::VIDEO_H265_CBR;
break;
case video::VIDEO_H265_VBR:
video_type = video::VIDEO_H265_VBR;
break;
default:
err::check_raise(err::ERR_RUNTIME, "Unsupported video type!");
break;
}
} else if (!strcmp(suffix, ".mp4")) {
switch (type) {
case video::VIDEO_H264: // fall through
case video::VIDEO_H264_MP4: // fall through
case video::VIDEO_H264_FLV:
video_type = video::VIDEO_H264_MP4;
case video::VIDEO_H264_CBR:
video_type = video::VIDEO_H264_CBR;
break;
case video::VIDEO_H264_VBR:
video_type = video::VIDEO_H264_VBR;
break;
case video::VIDEO_H265: // fall through
case video::VIDEO_H265_MP4:
video_type = video::VIDEO_H265_MP4;
case video::VIDEO_H265_CBR:
video_type = video::VIDEO_H265_CBR;
break;
case video::VIDEO_H265_VBR:
video_type = video::VIDEO_H265_VBR;
break;
default:
err::check_raise(err::ERR_RUNTIME, "Unsupported video type!");
@@ -77,8 +104,11 @@ namespace maix::video
} else if (!strcmp(suffix, ".flv")) {
switch (type) {
case video::VIDEO_H264: // fall through
case video::VIDEO_H264_FLV:
video_type = video::VIDEO_H264_FLV;
case video::VIDEO_H264_CBR:
video_type = video::VIDEO_H264_CBR;
break;
case video::VIDEO_H264_VBR:
video_type = video::VIDEO_H264_VBR;
break;
default:
err::check_raise(err::ERR_RUNTIME, "Unsupported video type!");
@@ -94,12 +124,13 @@ namespace maix::video
PAYLOAD_TYPE_E payload_type = PT_H264;
switch (video_type) {
case VIDEO_H264:
case VIDEO_H264_MP4:
case VIDEO_H264_FLV:
case VIDEO_H264_CBR:
case VIDEO_H264_VBR:
payload_type = PT_H264;
break;
case VIDEO_H265:
case VIDEO_H265_MP4:
case VIDEO_H265_CBR:
case VIDEO_H265_VBR:
payload_type = PT_H265;
break;
default:
@@ -112,12 +143,13 @@ namespace maix::video
enum AVCodecID codec_id = AV_CODEC_ID_NONE;
switch (video_type) {
case VIDEO_H264:
case VIDEO_H264_MP4:
case VIDEO_H264_FLV:
case VIDEO_H264_CBR:
case VIDEO_H264_VBR:
codec_id = AV_CODEC_ID_H264;
break;
case VIDEO_H265:
case VIDEO_H265_MP4:
case VIDEO_H265_CBR:
case VIDEO_H265_VBR:
codec_id = AV_CODEC_ID_HEVC;
break;
default:
@@ -247,6 +279,8 @@ namespace maix::video
if (_path.size() == 0) {
switch (_type) {
case VIDEO_H264:
case VIDEO_H264_CBR:
case VIDEO_H264_VBR:
{
mmf_venc_cfg_t cfg = {
.type = 2, //1, h265, 2, h264
@@ -258,8 +292,20 @@ namespace maix::video
.intput_fps = _framerate,
.output_fps = _framerate,
.bitrate = _bitrate / 1000,
.rc_mode = MMF_VENC_RCMODE_CBR,
};
switch (_type) {
case VIDEO_H264_CBR:
cfg.rc_mode = MMF_VENC_RCMODE_CBR;
break;
case VIDEO_H264_VBR:
cfg.rc_mode = MMF_VENC_RCMODE_VBR;
break;
default:
break;
}
if (0 != mmf_init_v2(true)) {
err::check_raise(err::ERR_RUNTIME, "init mmf failed!");
}
@@ -271,6 +317,8 @@ namespace maix::video
break;
}
case VIDEO_H265:
case VIDEO_H265_CBR:
case VIDEO_H265_VBR:
{
mmf_venc_cfg_t cfg = {
.type = 1, //1, h265, 2, h264
@@ -282,8 +330,20 @@ namespace maix::video
.intput_fps = _framerate,
.output_fps = _framerate,
.bitrate = _bitrate / 1000,
.rc_mode = MMF_VENC_RCMODE_CBR,
};
switch (_type) {
case VIDEO_H265_CBR:
cfg.rc_mode = MMF_VENC_RCMODE_CBR;
break;
case VIDEO_H265_VBR:
cfg.rc_mode = MMF_VENC_RCMODE_VBR;
break;
default:
break;
}
if (0 != mmf_init_v2(true)) {
err::check_raise(err::ERR_RUNTIME, "init mmf failed!");
}
@@ -344,13 +404,32 @@ namespace maix::video
.intput_fps = _framerate,
.output_fps = _framerate,
.bitrate = _bitrate / 1000,
.rc_mode = MMF_VENC_RCMODE_CBR,
};
if (venc_type == PT_H265) {
cfg.type = 1;
} else if (venc_type == PT_H264) {
cfg.type = 2;
}
switch (type) {
case VIDEO_H264_CBR:
cfg.rc_mode = MMF_VENC_RCMODE_CBR;
break;
case VIDEO_H264_VBR:
cfg.rc_mode = MMF_VENC_RCMODE_VBR;
break;
case VIDEO_H265_CBR:
cfg.rc_mode = MMF_VENC_RCMODE_CBR;
break;
case VIDEO_H265_VBR:
cfg.rc_mode = MMF_VENC_RCMODE_VBR;
break;
default:
break;
}
if (0 != mmf_init_v2(true)) {
err::check_raise(err::ERR_RUNTIME, "init mmf failed!");
}
@@ -368,46 +447,48 @@ namespace maix::video
SwrContext *swr_ctx = NULL;
AVPacket *audio_packet = NULL;
audio_packet = av_packet_alloc();
err::check_null_raise(audio_packet, "av_packet_alloc");
audio_packet->data = NULL;
audio_packet->size = 0;
if (_path.size() == 0) {
audio_packet = av_packet_alloc();
err::check_null_raise(audio_packet, "av_packet_alloc");
audio_packet->data = NULL;
audio_packet->size = 0;
int sample_rate = 48000;
int channels = 1;
int bitrate = 128000;
enum AVSampleFormat format = AV_SAMPLE_FMT_S16;
err::check_null_raise(audio_codec = avcodec_find_encoder(AV_CODEC_ID_AAC), "Could not find aac encoder");
err::check_null_raise(audio_stream = avformat_new_stream(outputFormatContext, NULL), "Could not allocate stream");
err::check_null_raise(audio_codec_ctx = avcodec_alloc_context3(audio_codec), "Could not allocate audio codec context");
audio_codec_ctx->codec_id = AV_CODEC_ID_AAC;
audio_codec_ctx->codec_type = AVMEDIA_TYPE_AUDIO;
audio_codec_ctx->sample_rate = sample_rate;
audio_codec_ctx->channels = channels;
audio_codec_ctx->channel_layout = av_get_default_channel_layout(audio_codec_ctx->channels);
audio_codec_ctx->sample_fmt = AV_SAMPLE_FMT_FLTP; // AAC编码需要浮点格式
audio_codec_ctx->time_base = (AVRational){1, sample_rate};
audio_codec_ctx->bit_rate = bitrate;
audio_stream->time_base = audio_codec_ctx->time_base;
err::check_bool_raise(avcodec_parameters_from_context(audio_stream->codecpar, audio_codec_ctx) >= 0, "avcodec_parameters_to_context");
err::check_bool_raise(avcodec_open2(audio_codec_ctx, audio_codec, NULL) >= 0, "audio_codec open failed");
int sample_rate = 48000;
int channels = 1;
int bitrate = 128000;
enum AVSampleFormat format = AV_SAMPLE_FMT_S16;
err::check_null_raise(audio_codec = avcodec_find_encoder(AV_CODEC_ID_AAC), "Could not find aac encoder");
err::check_null_raise(audio_stream = avformat_new_stream(outputFormatContext, NULL), "Could not allocate stream");
err::check_null_raise(audio_codec_ctx = avcodec_alloc_context3(audio_codec), "Could not allocate audio codec context");
audio_codec_ctx->codec_id = AV_CODEC_ID_AAC;
audio_codec_ctx->codec_type = AVMEDIA_TYPE_AUDIO;
audio_codec_ctx->sample_rate = sample_rate;
audio_codec_ctx->channels = channels;
audio_codec_ctx->channel_layout = av_get_default_channel_layout(audio_codec_ctx->channels);
audio_codec_ctx->sample_fmt = AV_SAMPLE_FMT_FLTP; // AAC编码需要浮点格式
audio_codec_ctx->time_base = (AVRational){1, sample_rate};
audio_codec_ctx->bit_rate = bitrate;
audio_stream->time_base = audio_codec_ctx->time_base;
err::check_bool_raise(avcodec_parameters_from_context(audio_stream->codecpar, audio_codec_ctx) >= 0, "avcodec_parameters_to_context");
err::check_bool_raise(avcodec_open2(audio_codec_ctx, audio_codec, NULL) >= 0, "audio_codec open failed");
swr_ctx = swr_alloc();
av_opt_set_int(swr_ctx, "in_channel_layout", audio_codec_ctx->channel_layout, 0);
av_opt_set_int(swr_ctx, "out_channel_layout", audio_codec_ctx->channel_layout, 0);
av_opt_set_int(swr_ctx, "in_sample_rate", audio_codec_ctx->sample_rate, 0);
av_opt_set_int(swr_ctx, "out_sample_rate", audio_codec_ctx->sample_rate, 0);
av_opt_set_sample_fmt(swr_ctx, "in_sample_fmt", format, 0);
av_opt_set_sample_fmt(swr_ctx, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
swr_init(swr_ctx);
swr_ctx = swr_alloc();
av_opt_set_int(swr_ctx, "in_channel_layout", audio_codec_ctx->channel_layout, 0);
av_opt_set_int(swr_ctx, "out_channel_layout", audio_codec_ctx->channel_layout, 0);
av_opt_set_int(swr_ctx, "in_sample_rate", audio_codec_ctx->sample_rate, 0);
av_opt_set_int(swr_ctx, "out_sample_rate", audio_codec_ctx->sample_rate, 0);
av_opt_set_sample_fmt(swr_ctx, "in_sample_fmt", format, 0);
av_opt_set_sample_fmt(swr_ctx, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
swr_init(swr_ctx);
int frame_size = audio_codec_ctx->frame_size;
audio_frame = av_frame_alloc();
audio_frame->nb_samples = frame_size;
audio_frame->channel_layout = audio_codec_ctx->channel_layout;
audio_frame->format = AV_SAMPLE_FMT_FLTP;
audio_frame->sample_rate = audio_codec_ctx->sample_rate;
av_frame_get_buffer(audio_frame, 0);
int frame_size = audio_codec_ctx->frame_size;
audio_frame = av_frame_alloc();
audio_frame->nb_samples = frame_size;
audio_frame->channel_layout = audio_codec_ctx->channel_layout;
audio_frame->format = AV_SAMPLE_FMT_FLTP;
audio_frame->sample_rate = audio_codec_ctx->sample_rate;
av_frame_get_buffer(audio_frame, 0);
}
err::check_bool_raise(avformat_write_header(outputFormatContext, NULL) >= 0, "avformat_write_header failed!");
@@ -425,21 +506,24 @@ namespace maix::video
param->frame_index = 0;
param->last_encode_ms = time::ticks_ms();
param->video_packet_list = new std::list<AVPacket *>;
bool is_raw_h264 = _path.find(".h264") != std::string::npos;
bool is_raw_h265 = _path.find(".h265") != std::string::npos;
switch (video_type) {
case VIDEO_H264:
param->copy_sps_pps_per_iframe = true;
break;
case VIDEO_H264_MP4:
param->copy_sps_pps_per_iframe = false;
case VIDEO_H264_CBR:
case VIDEO_H264_VBR:
param->copy_sps_pps_per_iframe = is_raw_h264 ? true : false;
break;
case VIDEO_H264_FLV:
param->copy_sps_pps_per_iframe = false;
break;
case VIDEO_H265:
param->copy_sps_pps_per_iframe = true;
break;
case VIDEO_H265_MP4:
param->copy_sps_pps_per_iframe = false;
case VIDEO_H265_CBR:
case VIDEO_H265_VBR:
param->copy_sps_pps_per_iframe = is_raw_h265 ? true : false;
break;
default:
err::check_raise(err::ERR_RUNTIME, "Unsupported video type!");
@@ -465,12 +549,16 @@ namespace maix::video
if (_path.size() == 0) {
switch (_type) {
case VIDEO_H264:
case VIDEO_H264_CBR:
case VIDEO_H264_VBR:
{
mmf_del_venc_channel(MMF_VENC_CHN);
mmf_deinit_v2(false);
break;
}
case VIDEO_H265:
case VIDEO_H265_CBR:
case VIDEO_H265_VBR:
{
mmf_del_venc_channel(MMF_VENC_CHN);
mmf_deinit_v2(false);
@@ -550,6 +638,8 @@ namespace maix::video
switch (_type) {
case VIDEO_H264:
case VIDEO_H264_CBR:
case VIDEO_H264_VBR:
{
if (img && img->data() != NULL) { // encode from image
if (img->data_size() > 2560 * 1440 * 3 / 2) {
@@ -768,6 +858,8 @@ namespace maix::video
break;
}
case VIDEO_H265:
case VIDEO_H265_CBR:
case VIDEO_H265_VBR:
{
if (img && img->data() != NULL) { // encode from image
if (img->data_size() > 2560 * 1440 * 3 / 2) {
@@ -967,6 +1059,8 @@ namespace maix::video
switch (_type) {
case VIDEO_H264:
case VIDEO_H264_CBR:
case VIDEO_H264_VBR:
{
if (_block) {
if (use_input_img) {
@@ -4031,21 +4125,24 @@ _exit:
packager->find_sps_pps = false;
packager->frame_index = 0;
packager->last_encode_ms = time::ticks_ms();
bool is_raw_h264 = path.find(".h264") != std::string::npos;
bool is_raw_h265 = path.find(".h265") != std::string::npos;
switch (video_type) {
case VIDEO_H264:
packager->copy_sps_pps_per_iframe = true;
break;
case VIDEO_H264_MP4:
packager->copy_sps_pps_per_iframe = false;
case VIDEO_H264_CBR:
case VIDEO_H264_VBR:
packager->copy_sps_pps_per_iframe = is_raw_h264 ? true : false;
break;
case VIDEO_H264_FLV:
packager->copy_sps_pps_per_iframe = false;
break;
case VIDEO_H265:
packager->copy_sps_pps_per_iframe = true;
break;
case VIDEO_H265_MP4:
packager->copy_sps_pps_per_iframe = false;
case VIDEO_H265_CBR:
case VIDEO_H265_VBR:
packager->copy_sps_pps_per_iframe = is_raw_h265 ? true : false;
break;
default:
err::check_raise(err::ERR_RUNTIME, "Unsupported video type!");

48
components/vision/port/maixcam/maix_webrtc_maixcam.cpp
View File

@@ -165,7 +165,6 @@ namespace maix::webrtc
bool bind_audio_recorder;
int encoder_bitrate;
int fps;
int gop;
MaixWebRTCServer *webrtc_server;
@@ -257,20 +256,37 @@ namespace maix::webrtc
param->status = WEBRTC_IDLE;
}
WebRTC::WebRTC(std::string ip, int port, int fps,
webrtc::WebRTCStreamType stream_type, int bitrate, int gop,
std::string signaling_ip, int signaling_port,
const std::string &stun_server,
bool http_server)
static video::VideoType get_video_type( maix::webrtc::WebRTCStreamType stream, maix::webrtc::WebRTCRCType rc)
{
using stream_type = maix::webrtc::WebRTCStreamType;
using rc_type = maix::webrtc::WebRTCRCType;
if (stream == stream_type::WEBRTC_STREAM_H264) {
if (rc == rc_type::WEBRTC_RC_CBR) { return video::VIDEO_H264_CBR; }
if (rc == rc_type::WEBRTC_RC_VBR) { return video::VIDEO_H264_VBR; }
}
if (stream == stream_type::WEBRTC_STREAM_H265) {
if (rc == rc_type::WEBRTC_RC_CBR) { return video::VIDEO_H265_CBR; }
if (rc == rc_type::WEBRTC_RC_VBR) { return video::VIDEO_H265_VBR; }
}
log::error("Unsupported video type stream=%d rc=%d", (int)stream, (int)rc);
return video::VIDEO_H264_CBR;
}
WebRTC::WebRTC(std::string ip, int port, webrtc::WebRTCStreamType stream_type,
webrtc::WebRTCRCType rc_type, int bitrate, int gop, std::string signaling_ip,
int signaling_port, const std::string &stun_server, bool http_server)
{
this->_ip = ip.size() ? ip : "0.0.0.0";
this->_port = port;
this->_signaling_ip = signaling_ip;
this->_signaling_port = signaling_port;
this->_stun_server = stun_server;
this->_fps = fps;
this->_gop = gop;
this->_stream_type = stream_type;
this->_rc_type = rc_type;
this->_is_start = false;
this->_video_thread = nullptr;
this->_http_server = http_server;
@@ -292,7 +308,6 @@ namespace maix::webrtc
param->bind_camera = false;
param->bind_audio_recorder = false;
param->encoder_bitrate = bitrate;
param->fps = fps;
param->gop = gop;
param->webrtc_server = nullptr;
@@ -353,17 +368,9 @@ namespace maix::webrtc
param->encoder = nullptr;
}
video::VideoType video_type;
if (this->_stream_type == maix::webrtc::WebRTCStreamType::WEBRTC_STREAM_H264) {
video_type = video::VIDEO_H264;
} else if (this->_stream_type == maix::webrtc::WebRTCStreamType::WEBRTC_STREAM_H265) {
video_type = video::VIDEO_H265;
} else {
log::error("Unsupported stream type: %d", this->_stream_type);
return err::ERR_ARGS;
}
video::VideoType video_type = get_video_type(this->_stream_type, this->_rc_type);
param->encoder = new video::Encoder("", param->camera->width(), param->camera->height(), image::Format::FMT_YVU420SP, video_type, param->fps, param->gop, param->encoder_bitrate);
param->encoder = new video::Encoder("", param->camera->width(), param->camera->height(), image::Format::FMT_YVU420SP, video_type, param->camera->fps(), param->gop, param->encoder_bitrate);
err::check_null_raise(param->encoder, "Create video encoder failed!");
if (param->bind_audio_recorder) {
@@ -376,7 +383,7 @@ namespace maix::webrtc
}
MaixWebRTCServerBuilder builder;
builder.set_ice_server("stun:stun.l.google.com:19302");
builder.set_ice_server(this->_stun_server);
if (this->_stream_type == maix::webrtc::WebRTCStreamType::WEBRTC_STREAM_H265) {
builder.set_video_codec(VideoCodec::H265);
@@ -541,6 +548,9 @@ namespace maix::webrtc
if (!get_ip((char *)"wlan0", new_ip)) {
ip_list.push_back("http://" + std::string(new_ip) + ":" + std::to_string(port));
}
if (!get_ip((char *)"tailscale0", new_ip)) {
ip_list.push_back("http://" + std::string(new_ip) + ":" + std::to_string(port));
}
} else {
ip_list.push_back("http://" + ip + ":" + std::to_string(port));
}

126
components/vision/port/maixcam2/maix_video_maixcam2.cpp
View File

@@ -81,19 +81,46 @@ namespace maix::video
}
if (!strcmp(suffix, ".h264")) {
video_type = video::VIDEO_H264;
switch (type) {
case video::VIDEO_H264: // fall through
case video::VIDEO_H264_CBR:
video_type = video::VIDEO_H264_CBR;
break;
case video::VIDEO_H264_VBR:
video_type = video::VIDEO_H264_VBR;
break;
default:
err::check_raise(err::ERR_RUNTIME, "Unsupported video type!");
break;
}
} else if (!strcmp(suffix, ".h265")) {
video_type = video::VIDEO_H265;
switch (type) {
case video::VIDEO_H265: // fall through
case video::VIDEO_H265_CBR:
video_type = video::VIDEO_H265_CBR;
break;
case video::VIDEO_H265_VBR:
video_type = video::VIDEO_H265_VBR;
break;
default:
err::check_raise(err::ERR_RUNTIME, "Unsupported video type!");
break;
}
} else if (!strcmp(suffix, ".mp4")) {
switch (type) {
case video::VIDEO_H264: // fall through
case video::VIDEO_H264_MP4: // fall through
case video::VIDEO_H264_FLV:
video_type = video::VIDEO_H264_MP4;
case video::VIDEO_H264_CBR:
video_type = video::VIDEO_H264_CBR;
break;
case video::VIDEO_H264_VBR:
video_type = video::VIDEO_H264_VBR;
break;
case video::VIDEO_H265: // fall through
case video::VIDEO_H265_MP4:
video_type = video::VIDEO_H265_MP4;
case video::VIDEO_H265_CBR:
video_type = video::VIDEO_H265_CBR;
break;
case video::VIDEO_H265_VBR:
video_type = video::VIDEO_H265_VBR;
break;
default:
err::check_raise(err::ERR_RUNTIME, "Unsupported video type!");
@@ -102,8 +129,11 @@ namespace maix::video
} else if (!strcmp(suffix, ".flv")) {
switch (type) {
case video::VIDEO_H264: // fall through
case video::VIDEO_H264_FLV:
video_type = video::VIDEO_H264_FLV;
case video::VIDEO_H264_CBR:
video_type = video::VIDEO_H264_CBR;
break;
case video::VIDEO_H264_VBR:
video_type = video::VIDEO_H264_VBR;
break;
default:
err::check_raise(err::ERR_RUNTIME, "Unsupported video type!");
@@ -120,12 +150,13 @@ namespace maix::video
enum AVCodecID codec_id = AV_CODEC_ID_NONE;
switch (video_type) {
case VIDEO_H264:
case VIDEO_H264_MP4:
case VIDEO_H264_FLV:
case VIDEO_H264_CBR:
case VIDEO_H264_VBR:
codec_id = AV_CODEC_ID_H264;
break;
case VIDEO_H265:
case VIDEO_H265_MP4:
case VIDEO_H265_CBR:
case VIDEO_H265_VBR:
codec_id = AV_CODEC_ID_HEVC;
break;
default:
@@ -313,21 +344,24 @@ namespace maix::video
param->frame_index = 0;
param->last_encode_ms = time::ticks_ms();
param->video_packet_list = new std::list<AVPacket *>;
bool is_raw_h264 = _path.find(".h264") != std::string::npos;
bool is_raw_h265 = _path.find(".h265") != std::string::npos;
switch (video_type) {
case VIDEO_H264:
param->copy_sps_pps_per_iframe = true;
break;
case VIDEO_H264_MP4:
param->copy_sps_pps_per_iframe = false;
case VIDEO_H264_CBR:
case VIDEO_H264_VBR:
param->copy_sps_pps_per_iframe = is_raw_h264 ? true : false;
break;
case VIDEO_H264_FLV:
param->copy_sps_pps_per_iframe = false;
break;
case VIDEO_H265:
param->copy_sps_pps_per_iframe = true;
break;
case VIDEO_H265_MP4:
param->copy_sps_pps_per_iframe = false;
case VIDEO_H265_CBR:
case VIDEO_H265_VBR:
param->copy_sps_pps_per_iframe = is_raw_h265 ? true : false;
break;
default:
err::check_raise(err::ERR_RUNTIME, "Unsupported video type!");
@@ -336,12 +370,12 @@ namespace maix::video
maixcam2::ax_venc_param_t cfg = {0};
switch (video_type) {
case VIDEO_H264:
case VIDEO_H264_MP4:
case VIDEO_H264_FLV:
case VIDEO_H264_CBR:
cfg.w = width;
cfg.h = height;
cfg.fmt = maixcam2::get_ax_fmt_from_maix(format);
cfg.type = maixcam2::AX_VENC_TYPE_H264;
cfg.rc_mode = maixcam2::AX_VENC_RCMODE_CBR;
cfg.h264.bitrate = bitrate / 1000;
cfg.h264.input_fps = framerate;
cfg.h264.output_fps = framerate;
@@ -356,12 +390,31 @@ namespace maix::video
cfg.h264.max_iprop = 40;
cfg.h264.first_frame_start_qp = -1;
break;
case VIDEO_H264_VBR:
cfg.w = width;
cfg.h = height;
cfg.fmt = maixcam2::get_ax_fmt_from_maix(format);
cfg.type = maixcam2::AX_VENC_TYPE_H264;
cfg.rc_mode = maixcam2::AX_VENC_RCMODE_VBR;
cfg.h264.bitrate = bitrate / 1000;
cfg.h264.input_fps = framerate;
cfg.h264.output_fps = framerate;
cfg.h264.gop = gop;
cfg.h264.intra_qp_delta = -2;
cfg.h264.de_breath_qp_delta = -2;
cfg.h264.min_qp = 31;
cfg.h264.max_qp = 46;
cfg.h264.min_iqp = 31;
cfg.h264.max_iqp = 46;
cfg.h264.first_frame_start_qp = -1;
break;
case VIDEO_H265:
case VIDEO_H265_MP4:
case VIDEO_H265_CBR:
cfg.w = width;
cfg.h = height;
cfg.fmt = maixcam2::get_ax_fmt_from_maix(format);
cfg.type = maixcam2::AX_VENC_TYPE_H265;
cfg.rc_mode = maixcam2::AX_VENC_RCMODE_CBR;
cfg.h265.bitrate = bitrate / 1000;
cfg.h265.input_fps = framerate;
cfg.h265.output_fps = framerate;
@@ -376,10 +429,25 @@ namespace maix::video
cfg.h265.max_iprop = 40;
cfg.h265.first_frame_start_qp = -1;
cfg.h265.qp_delta_rgn = 10;
cfg.h265.qp_map_type = AX_VENC_QPMAP_QP_DISABLE;
cfg.h265.qp_map_blk_type = AX_VENC_QPMAP_BLOCK_DISABLE;
cfg.h265.qp_map_block_unit = AX_VENC_QPMAP_BLOCK_UNIT_64x64;
cfg.h265.ctb_rc_mode = AX_VENC_RC_CTBRC_DISABLE;
break;
case VIDEO_H265_VBR:
cfg.w = width;
cfg.h = height;
cfg.fmt = maixcam2::get_ax_fmt_from_maix(format);
cfg.rc_mode = maixcam2::AX_VENC_RCMODE_VBR;
cfg.type = maixcam2::AX_VENC_TYPE_H265;
cfg.h265.bitrate = bitrate / 1000;
cfg.h265.input_fps = framerate;
cfg.h265.output_fps = framerate;
cfg.h265.gop = gop;
cfg.h265.intra_qp_delta = -2;
cfg.h265.de_breath_qp_delta = -2;
cfg.h265.min_qp = 31;
cfg.h265.max_qp = 46;
cfg.h265.min_iqp = 31;
cfg.h265.max_iqp = 46;
cfg.h265.first_frame_start_qp = -1;
cfg.h265.qp_delta_rgn = 10;
break;
default:
err::check_raise(err::ERR_RUNTIME, "unsupport stream type!");
@@ -2050,10 +2118,12 @@ __vdec_exit:
::close(this->_fd);
switch (this->_video_type) {
case VIDEO_ENC_H265_CBR:
case VIDEO_H265_CBR:
case VIDEO_H265_VBR:
// do nothing
break;
case VIDEO_ENC_MP4_CBR:
case VIDEO_H264_CBR:
case VIDEO_H264_VBR:
{
char cmd[128];
snprintf(cmd, sizeof(cmd), "ffmpeg -loglevel quiet -i %s -c:v copy -c:a copy %s -y", this->_tmp_path.c_str(), this->_path.c_str());

45
components/vision/port/maixcam2/maix_webrtc_maixcam2.cpp
View File

@@ -66,7 +66,6 @@ namespace maix::webrtc
bool bind_audio_recorder;
int encoder_bitrate;
int fps;
int gop;
MaixWebRTCServer *webrtc_server;
@@ -151,20 +150,37 @@ namespace maix::webrtc
param->status = WEBRTC_IDLE;
}
WebRTC::WebRTC(std::string ip, int port, int fps,
webrtc::WebRTCStreamType stream_type, int bitrate, int gop,
std::string signaling_ip, int signaling_port,
const std::string &stun_server,
bool http_server)
static video::VideoType get_video_type( maix::webrtc::WebRTCStreamType stream, maix::webrtc::WebRTCRCType rc)
{
using stream_type = maix::webrtc::WebRTCStreamType;
using rc_type = maix::webrtc::WebRTCRCType;
if (stream == stream_type::WEBRTC_STREAM_H264) {
if (rc == rc_type::WEBRTC_RC_CBR) { return video::VIDEO_H264_CBR; }
if (rc == rc_type::WEBRTC_RC_VBR) { return video::VIDEO_H264_VBR; }
}
if (stream == stream_type::WEBRTC_STREAM_H265) {
if (rc == rc_type::WEBRTC_RC_CBR) { return video::VIDEO_H265_CBR; }
if (rc == rc_type::WEBRTC_RC_VBR) { return video::VIDEO_H265_VBR; }
}
log::error("Unsupported video type stream=%d rc=%d", (int)stream, (int)rc);
return video::VIDEO_H264_CBR;
}
WebRTC::WebRTC(std::string ip, int port, webrtc::WebRTCStreamType stream_type,
webrtc::WebRTCRCType rc_type, int bitrate, int gop, std::string signaling_ip,
int signaling_port, const std::string &stun_server, bool http_server)
{
this->_ip = ip.size() ? ip : "0.0.0.0";
this->_port = port;
this->_signaling_ip = signaling_ip;
this->_signaling_port = signaling_port;
this->_stun_server = stun_server;
this->_fps = fps;
this->_gop = gop;
this->_stream_type = stream_type;
this->_rc_type = rc_type;
this->_is_start = false;
this->_video_thread = nullptr;
this->_http_server = http_server;
@@ -186,7 +202,6 @@ namespace maix::webrtc
param->bind_camera = false;
param->bind_audio_recorder = false;
param->encoder_bitrate = bitrate;
param->fps = fps;
param->gop = gop;
param->webrtc_server = nullptr;
@@ -247,17 +262,9 @@ namespace maix::webrtc
param->encoder = nullptr;
}
video::VideoType video_type;
if (this->_stream_type == maix::webrtc::WebRTCStreamType::WEBRTC_STREAM_H264) {
video_type = video::VIDEO_H264;
} else if (this->_stream_type == maix::webrtc::WebRTCStreamType::WEBRTC_STREAM_H265) {
video_type = video::VIDEO_H265;
} else {
log::error("Unsupported stream type: %d", this->_stream_type);
return err::ERR_ARGS;
}
video::VideoType video_type = get_video_type(this->_stream_type, this->_rc_type);
param->encoder = new video::Encoder("", param->camera->width(), param->camera->height(), image::Format::FMT_YVU420SP, video_type, param->fps, param->gop, param->encoder_bitrate);
param->encoder = new video::Encoder("", param->camera->width(), param->camera->height(), image::Format::FMT_YVU420SP, video_type, param->camera->fps(), param->gop, param->encoder_bitrate);
err::check_null_raise(param->encoder, "Create video encoder failed!");
if (param->bind_audio_recorder) {
@@ -270,7 +277,7 @@ namespace maix::webrtc
}
MaixWebRTCServerBuilder builder;
builder.set_ice_server("stun:stun.l.google.com:19302");
builder.set_ice_server(this->_stun_server);
if (this->_stream_type == maix::webrtc::WebRTCStreamType::WEBRTC_STREAM_H265) {
builder.set_video_codec(VideoCodec::H265);