diff --git a/components/3rd_party/datachannel/CMakeLists.txt b/components/3rd_party/datachannel/CMakeLists.txt index 878208f4..6cd8ccc7 100644 --- a/components/3rd_party/datachannel/CMakeLists.txt +++ b/components/3rd_party/datachannel/CMakeLists.txt @@ -19,16 +19,18 @@ if (compile_from_src) aux_source_directory("${srcs_path}/src/impl" ADD_SRCS) list(APPEND ADD_REQUIREMENTS openssl json libjuice libsrtp plog usrsctp cpp-httplib) list(APPEND ADD_DEFINITIONS_PRIVATE -DJUICE_STATIC -DRTC_ENABLE_MEDIA=1 -DRTC_ENABLE_WEBSOCKET=1 -DRTC_EXPORTS -DRTC_SYSTEM_JUICE=0 -DRTC_SYSTEM_SRTP=0 -DUSE_GNUTLS=0 -DUSE_NICE=0 -D_GNU_SOURCE -Ddatachannel_EXPORTS -DNDEBUG -w) + + register_component(DYNAMIC) else() list(APPEND ADD_REQUIREMENTS json cpp-httplib) if (PLATFORM_MAIXCAM) - list(APPEND ADD_DYNAMIC_LIB "so/maixcam/libdatachannel.so") + list(APPEND ADD_DYNAMIC_LIB "lib/maixcam/libdatachannel.so") elseif(PLATFORM_MAIXCAM2) - list(APPEND ADD_DYNAMIC_LIB "so/maixcam2/libdatachannel.so") + list(APPEND ADD_DYNAMIC_LIB "lib/maixcam2/libdatachannel.so") else() list(APPEND ADD_REQUIREMENTS libdatachannel) endif() + + register_component() endif() -register_component() - diff --git a/components/maixcam_lib/include/maixcam/sophgo_middleware.hpp b/components/maixcam_lib/include/maixcam/sophgo_middleware.hpp index a2aef7af..5ecdada1 100644 --- a/components/maixcam_lib/include/maixcam/sophgo_middleware.hpp +++ b/components/maixcam_lib/include/maixcam/sophgo_middleware.hpp @@ -41,6 +41,14 @@ typedef struct { int count; } mmf_h265_stream_t; +typedef enum +{ + MMF_VENC_RCMODE_CBR = 0, + MMF_VENC_RCMODE_VBR, + MMF_VENC_RCMODE_FIXQP, + MMF_VENC_RCMODE_MAX, +} mmf_venc_rc_mode_e; + typedef struct { uint8_t type; // 0, jpg; 1, h265; 2, h264 int w; @@ -51,6 +59,7 @@ typedef struct { int intput_fps; // h264/h265 int output_fps; // h264/h265 int bitrate; // h264/h265 + mmf_venc_rc_mode_e rc_mode; // h264/h265 } mmf_venc_cfg_t; typedef struct { diff --git a/components/maixcam_lib/include/maixcam2/ax_middleware.hpp b/components/maixcam_lib/include/maixcam2/ax_middleware.hpp index c58cf430..58d9bf81 100644 --- a/components/maixcam_lib/include/maixcam2/ax_middleware.hpp +++ b/components/maixcam_lib/include/maixcam2/ax_middleware.hpp @@ -59,6 +59,14 @@ namespace maix::middleware::maixcam2 { AX_VENC_TYPE_MJPG, } ax_venc_type_e; + typedef enum + { + AX_VENC_RCMODE_CBR = 0, + AX_VENC_RCMODE_VBR, + AX_VENC_RCMODE_FIXQP, + AX_VENC_RCMODE_MAX, + } ax_venc_rc_mode_e; + typedef enum { AX_VDEC_TYPE_JPG = 0, AX_VDEC_TYPE_H264, @@ -89,6 +97,7 @@ namespace maix::middleware::maixcam2 { typedef struct { bool en; ax_venc_type_e type; + ax_venc_rc_mode_e rc_mode; int w; int h; AX_IMG_FORMAT_E fmt; diff --git a/components/vision/include/maix_video.hpp b/components/vision/include/maix_video.hpp index af007a32..25bdebcc 100644 --- a/components/vision/include/maix_video.hpp +++ b/components/vision/include/maix_video.hpp @@ -31,16 +31,15 @@ namespace maix::video VIDEO_ENC_MP4_CBR, // Deprecated VIDEO_DEC_H265_CBR, // Deprecated VIDEO_DEC_MP4_CBR, // Deprecated - VIDEO_H264_CBR, // Deprecated - VIDEO_H265_CBR, // Deprecated VIDEO_H264_CBR_MP4, // Deprecated VIDEO_H265_CBR_MP4, // Deprecated VIDEO_H264, - VIDEO_H264_MP4, - VIDEO_H264_FLV, + VIDEO_H264_CBR, + VIDEO_H264_VBR, VIDEO_H265, - VIDEO_H265_MP4, + VIDEO_H265_CBR, + VIDEO_H265_VBR, }; /** diff --git a/components/vision/include/maix_webrtc.hpp b/components/vision/include/maix_webrtc.hpp index f378ef3b..030a4733 100644 --- a/components/vision/include/maix_webrtc.hpp +++ b/components/vision/include/maix_webrtc.hpp @@ -28,6 +28,17 @@ namespace maix::webrtc WEBRTC_STREAM_H265, }; + /** + * The rc type of webrtc + * @maixpy maix.webrtc.WebRTCRCType + */ + enum class WebRTCRCType + { + WEBRTC_RC_NONE = 0, // format invalid + WEBRTC_RC_CBR, + WEBRTC_RC_VBR, + }; + /** * Region class * @maixpy maix.webrtc.Region @@ -86,18 +97,20 @@ namespace maix::webrtc * @brief Construct a new WebRTC object * @param ip listen ip * @param port listen port - * @param fps video fps * @param stream_type stream type + * @param rc_type rc type * @param bitrate video bitrate + * @param gop video gop * @param signaling_ip signaling server bind ip * @param signaling_port signaling server bind port - * @param http_server whether to enable the HTTP server * @param stun_server STUN server address + * @param http_server whether to enable the HTTP server * @maixpy maix.webrtc.WebRTC.__init__ * @maixcdk maix.webrtc.WebRTC.WebRTC */ - WebRTC(std::string ip = std::string(), int port = 8000, int fps = 60, + WebRTC(std::string ip = std::string(), int port = 8000, webrtc::WebRTCStreamType stream_type = webrtc::WebRTCStreamType::WEBRTC_STREAM_H264, + webrtc::WebRTCRCType rc_type = webrtc::WebRTCRCType::WEBRTC_RC_CBR, int bitrate = 3000 * 1000, int gop = 60, std::string signaling_ip = std::string(), int signaling_port = 8001, const std::string &stun_server = "stun:stun.l.google.com:19302", bool http_server = true); @@ -250,9 +263,9 @@ namespace maix::webrtc int _port; std::string _signaling_ip; int _signaling_port; - int _fps; int _gop; webrtc::WebRTCStreamType _stream_type; + webrtc::WebRTCRCType _rc_type; bool _is_start; thread::Thread *_video_thread; std::string _stun_server; diff --git a/components/vision/port/linux/maix_webrtc_linux.cpp b/components/vision/port/linux/maix_webrtc_linux.cpp index dfa4c416..fa7d9767 100644 --- a/components/vision/port/linux/maix_webrtc_linux.cpp +++ b/components/vision/port/linux/maix_webrtc_linux.cpp @@ -34,17 +34,15 @@ namespace maix::webrtc return err::ERR_NOT_IMPL; } - WebRTC::WebRTC(std::string ip, int port, int fps, - webrtc::WebRTCStreamType stream_type, int bitrate, int gop, - std::string signaling_ip, int signaling_port, - const std::string &stun_server, - bool http_server) + WebRTC::WebRTC(std::string ip, int port, webrtc::WebRTCStreamType stream_type, + webrtc::WebRTCRCType rc_type, int bitrate, int gop, std::string signaling_ip, + int signaling_port, const std::string &stun_server, bool http_server) { (void)ip; (void)port; - (void)fps; (void)gop; (void)stream_type; + (void)rc_type; (void)bitrate; (void)signaling_ip; (void)signaling_port; diff --git a/components/vision/port/maixcam/maix_video_mmf.cpp b/components/vision/port/maixcam/maix_video_mmf.cpp index d29e8ab7..ea220b35 100644 --- a/components/vision/port/maixcam/maix_video_mmf.cpp +++ b/components/vision/port/maixcam/maix_video_mmf.cpp @@ -56,19 +56,46 @@ namespace maix::video } if (!strcmp(suffix, ".h264")) { - video_type = video::VIDEO_H264; + switch (type) { + case video::VIDEO_H264: // fall through + case video::VIDEO_H264_CBR: + video_type = video::VIDEO_H264_CBR; + break; + case video::VIDEO_H264_VBR: + video_type = video::VIDEO_H264_VBR; + break; + default: + err::check_raise(err::ERR_RUNTIME, "Unsupported video type!"); + break; + } } else if (!strcmp(suffix, ".h265")) { - video_type = video::VIDEO_H264; + switch (type) { + case video::VIDEO_H265: // fall through + case video::VIDEO_H265_CBR: + video_type = video::VIDEO_H265_CBR; + break; + case video::VIDEO_H265_VBR: + video_type = video::VIDEO_H265_VBR; + break; + default: + err::check_raise(err::ERR_RUNTIME, "Unsupported video type!"); + break; + } } else if (!strcmp(suffix, ".mp4")) { switch (type) { case video::VIDEO_H264: // fall through - case video::VIDEO_H264_MP4: // fall through - case video::VIDEO_H264_FLV: - video_type = video::VIDEO_H264_MP4; + case video::VIDEO_H264_CBR: + video_type = video::VIDEO_H264_CBR; + break; + case video::VIDEO_H264_VBR: + video_type = video::VIDEO_H264_VBR; break; case video::VIDEO_H265: // fall through - case video::VIDEO_H265_MP4: - video_type = video::VIDEO_H265_MP4; + case video::VIDEO_H265_CBR: + video_type = video::VIDEO_H265_CBR; + break; + case video::VIDEO_H265_VBR: + video_type = video::VIDEO_H265_VBR; break; default: err::check_raise(err::ERR_RUNTIME, "Unsupported video type!"); @@ -77,8 +104,11 @@ namespace maix::video } else if (!strcmp(suffix, ".flv")) { switch (type) { case video::VIDEO_H264: // fall through - case video::VIDEO_H264_FLV: - video_type = video::VIDEO_H264_FLV; + case video::VIDEO_H264_CBR: + video_type = video::VIDEO_H264_CBR; + break; + case video::VIDEO_H264_VBR: + video_type = video::VIDEO_H264_VBR; break; default: err::check_raise(err::ERR_RUNTIME, "Unsupported video type!"); @@ -94,12 +124,13 @@ namespace maix::video PAYLOAD_TYPE_E payload_type = PT_H264; switch (video_type) { case VIDEO_H264: - case VIDEO_H264_MP4: - case VIDEO_H264_FLV: + case VIDEO_H264_CBR: + case VIDEO_H264_VBR: payload_type = PT_H264; break; case VIDEO_H265: - case VIDEO_H265_MP4: + case VIDEO_H265_CBR: + case VIDEO_H265_VBR: payload_type = PT_H265; break; default: @@ -112,12 +143,13 @@ namespace maix::video enum AVCodecID codec_id = AV_CODEC_ID_NONE; switch (video_type) { case VIDEO_H264: - case VIDEO_H264_MP4: - case VIDEO_H264_FLV: + case VIDEO_H264_CBR: + case VIDEO_H264_VBR: codec_id = AV_CODEC_ID_H264; break; case VIDEO_H265: - case VIDEO_H265_MP4: + case VIDEO_H265_CBR: + case VIDEO_H265_VBR: codec_id = AV_CODEC_ID_HEVC; break; default: @@ -247,6 +279,8 @@ namespace maix::video if (_path.size() == 0) { switch (_type) { case VIDEO_H264: + case VIDEO_H264_CBR: + case VIDEO_H264_VBR: { mmf_venc_cfg_t cfg = { .type = 2, //1, h265, 2, h264 @@ -258,8 +292,20 @@ namespace maix::video .intput_fps = _framerate, .output_fps = _framerate, .bitrate = _bitrate / 1000, + .rc_mode = MMF_VENC_RCMODE_CBR, }; + switch (_type) { + case VIDEO_H264_CBR: + cfg.rc_mode = MMF_VENC_RCMODE_CBR; + break; + case VIDEO_H264_VBR: + cfg.rc_mode = MMF_VENC_RCMODE_VBR; + break; + default: + break; + } + if (0 != mmf_init_v2(true)) { err::check_raise(err::ERR_RUNTIME, "init mmf failed!"); } @@ -271,6 +317,8 @@ namespace maix::video break; } case VIDEO_H265: + case VIDEO_H265_CBR: + case VIDEO_H265_VBR: { mmf_venc_cfg_t cfg = { .type = 1, //1, h265, 2, h264 @@ -282,8 +330,20 @@ namespace maix::video .intput_fps = _framerate, .output_fps = _framerate, .bitrate = _bitrate / 1000, + .rc_mode = MMF_VENC_RCMODE_CBR, }; + switch (_type) { + case VIDEO_H265_CBR: + cfg.rc_mode = MMF_VENC_RCMODE_CBR; + break; + case VIDEO_H265_VBR: + cfg.rc_mode = MMF_VENC_RCMODE_VBR; + break; + default: + break; + } + if (0 != mmf_init_v2(true)) { err::check_raise(err::ERR_RUNTIME, "init mmf failed!"); } @@ -344,13 +404,32 @@ namespace maix::video .intput_fps = _framerate, .output_fps = _framerate, .bitrate = _bitrate / 1000, + .rc_mode = MMF_VENC_RCMODE_CBR, }; + if (venc_type == PT_H265) { cfg.type = 1; } else if (venc_type == PT_H264) { cfg.type = 2; } + switch (type) { + case VIDEO_H264_CBR: + cfg.rc_mode = MMF_VENC_RCMODE_CBR; + break; + case VIDEO_H264_VBR: + cfg.rc_mode = MMF_VENC_RCMODE_VBR; + break; + case VIDEO_H265_CBR: + cfg.rc_mode = MMF_VENC_RCMODE_CBR; + break; + case VIDEO_H265_VBR: + cfg.rc_mode = MMF_VENC_RCMODE_VBR; + break; + default: + break; + } + if (0 != mmf_init_v2(true)) { err::check_raise(err::ERR_RUNTIME, "init mmf failed!"); } @@ -368,46 +447,48 @@ namespace maix::video SwrContext *swr_ctx = NULL; AVPacket *audio_packet = NULL; - audio_packet = av_packet_alloc(); - err::check_null_raise(audio_packet, "av_packet_alloc"); - audio_packet->data = NULL; - audio_packet->size = 0; + if (_path.size() == 0) { + audio_packet = av_packet_alloc(); + err::check_null_raise(audio_packet, "av_packet_alloc"); + audio_packet->data = NULL; + audio_packet->size = 0; - int sample_rate = 48000; - int channels = 1; - int bitrate = 128000; - enum AVSampleFormat format = AV_SAMPLE_FMT_S16; - err::check_null_raise(audio_codec = avcodec_find_encoder(AV_CODEC_ID_AAC), "Could not find aac encoder"); - err::check_null_raise(audio_stream = avformat_new_stream(outputFormatContext, NULL), "Could not allocate stream"); - err::check_null_raise(audio_codec_ctx = avcodec_alloc_context3(audio_codec), "Could not allocate audio codec context"); - audio_codec_ctx->codec_id = AV_CODEC_ID_AAC; - audio_codec_ctx->codec_type = AVMEDIA_TYPE_AUDIO; - audio_codec_ctx->sample_rate = sample_rate; - audio_codec_ctx->channels = channels; - audio_codec_ctx->channel_layout = av_get_default_channel_layout(audio_codec_ctx->channels); - audio_codec_ctx->sample_fmt = AV_SAMPLE_FMT_FLTP; // AAC编码需要浮点格式 - audio_codec_ctx->time_base = (AVRational){1, sample_rate}; - audio_codec_ctx->bit_rate = bitrate; - audio_stream->time_base = audio_codec_ctx->time_base; - err::check_bool_raise(avcodec_parameters_from_context(audio_stream->codecpar, audio_codec_ctx) >= 0, "avcodec_parameters_to_context"); - err::check_bool_raise(avcodec_open2(audio_codec_ctx, audio_codec, NULL) >= 0, "audio_codec open failed"); + int sample_rate = 48000; + int channels = 1; + int bitrate = 128000; + enum AVSampleFormat format = AV_SAMPLE_FMT_S16; + err::check_null_raise(audio_codec = avcodec_find_encoder(AV_CODEC_ID_AAC), "Could not find aac encoder"); + err::check_null_raise(audio_stream = avformat_new_stream(outputFormatContext, NULL), "Could not allocate stream"); + err::check_null_raise(audio_codec_ctx = avcodec_alloc_context3(audio_codec), "Could not allocate audio codec context"); + audio_codec_ctx->codec_id = AV_CODEC_ID_AAC; + audio_codec_ctx->codec_type = AVMEDIA_TYPE_AUDIO; + audio_codec_ctx->sample_rate = sample_rate; + audio_codec_ctx->channels = channels; + audio_codec_ctx->channel_layout = av_get_default_channel_layout(audio_codec_ctx->channels); + audio_codec_ctx->sample_fmt = AV_SAMPLE_FMT_FLTP; // AAC编码需要浮点格式 + audio_codec_ctx->time_base = (AVRational){1, sample_rate}; + audio_codec_ctx->bit_rate = bitrate; + audio_stream->time_base = audio_codec_ctx->time_base; + err::check_bool_raise(avcodec_parameters_from_context(audio_stream->codecpar, audio_codec_ctx) >= 0, "avcodec_parameters_to_context"); + err::check_bool_raise(avcodec_open2(audio_codec_ctx, audio_codec, NULL) >= 0, "audio_codec open failed"); - swr_ctx = swr_alloc(); - av_opt_set_int(swr_ctx, "in_channel_layout", audio_codec_ctx->channel_layout, 0); - av_opt_set_int(swr_ctx, "out_channel_layout", audio_codec_ctx->channel_layout, 0); - av_opt_set_int(swr_ctx, "in_sample_rate", audio_codec_ctx->sample_rate, 0); - av_opt_set_int(swr_ctx, "out_sample_rate", audio_codec_ctx->sample_rate, 0); - av_opt_set_sample_fmt(swr_ctx, "in_sample_fmt", format, 0); - av_opt_set_sample_fmt(swr_ctx, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0); - swr_init(swr_ctx); + swr_ctx = swr_alloc(); + av_opt_set_int(swr_ctx, "in_channel_layout", audio_codec_ctx->channel_layout, 0); + av_opt_set_int(swr_ctx, "out_channel_layout", audio_codec_ctx->channel_layout, 0); + av_opt_set_int(swr_ctx, "in_sample_rate", audio_codec_ctx->sample_rate, 0); + av_opt_set_int(swr_ctx, "out_sample_rate", audio_codec_ctx->sample_rate, 0); + av_opt_set_sample_fmt(swr_ctx, "in_sample_fmt", format, 0); + av_opt_set_sample_fmt(swr_ctx, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0); + swr_init(swr_ctx); - int frame_size = audio_codec_ctx->frame_size; - audio_frame = av_frame_alloc(); - audio_frame->nb_samples = frame_size; - audio_frame->channel_layout = audio_codec_ctx->channel_layout; - audio_frame->format = AV_SAMPLE_FMT_FLTP; - audio_frame->sample_rate = audio_codec_ctx->sample_rate; - av_frame_get_buffer(audio_frame, 0); + int frame_size = audio_codec_ctx->frame_size; + audio_frame = av_frame_alloc(); + audio_frame->nb_samples = frame_size; + audio_frame->channel_layout = audio_codec_ctx->channel_layout; + audio_frame->format = AV_SAMPLE_FMT_FLTP; + audio_frame->sample_rate = audio_codec_ctx->sample_rate; + av_frame_get_buffer(audio_frame, 0); + } err::check_bool_raise(avformat_write_header(outputFormatContext, NULL) >= 0, "avformat_write_header failed!"); @@ -425,21 +506,24 @@ namespace maix::video param->frame_index = 0; param->last_encode_ms = time::ticks_ms(); param->video_packet_list = new std::list; + + bool is_raw_h264 = _path.find(".h264") != std::string::npos; + bool is_raw_h265 = _path.find(".h265") != std::string::npos; + switch (video_type) { case VIDEO_H264: param->copy_sps_pps_per_iframe = true; break; - case VIDEO_H264_MP4: - param->copy_sps_pps_per_iframe = false; + case VIDEO_H264_CBR: + case VIDEO_H264_VBR: + param->copy_sps_pps_per_iframe = is_raw_h264 ? true : false; break; - case VIDEO_H264_FLV: - param->copy_sps_pps_per_iframe = false; - break; case VIDEO_H265: param->copy_sps_pps_per_iframe = true; break; - case VIDEO_H265_MP4: - param->copy_sps_pps_per_iframe = false; + case VIDEO_H265_CBR: + case VIDEO_H265_VBR: + param->copy_sps_pps_per_iframe = is_raw_h265 ? true : false; break; default: err::check_raise(err::ERR_RUNTIME, "Unsupported video type!"); @@ -465,12 +549,16 @@ namespace maix::video if (_path.size() == 0) { switch (_type) { case VIDEO_H264: + case VIDEO_H264_CBR: + case VIDEO_H264_VBR: { mmf_del_venc_channel(MMF_VENC_CHN); mmf_deinit_v2(false); break; } case VIDEO_H265: + case VIDEO_H265_CBR: + case VIDEO_H265_VBR: { mmf_del_venc_channel(MMF_VENC_CHN); mmf_deinit_v2(false); @@ -550,6 +638,8 @@ namespace maix::video switch (_type) { case VIDEO_H264: + case VIDEO_H264_CBR: + case VIDEO_H264_VBR: { if (img && img->data() != NULL) { // encode from image if (img->data_size() > 2560 * 1440 * 3 / 2) { @@ -768,6 +858,8 @@ namespace maix::video break; } case VIDEO_H265: + case VIDEO_H265_CBR: + case VIDEO_H265_VBR: { if (img && img->data() != NULL) { // encode from image if (img->data_size() > 2560 * 1440 * 3 / 2) { @@ -967,6 +1059,8 @@ namespace maix::video switch (_type) { case VIDEO_H264: + case VIDEO_H264_CBR: + case VIDEO_H264_VBR: { if (_block) { if (use_input_img) { @@ -4031,21 +4125,24 @@ _exit: packager->find_sps_pps = false; packager->frame_index = 0; packager->last_encode_ms = time::ticks_ms(); + + bool is_raw_h264 = path.find(".h264") != std::string::npos; + bool is_raw_h265 = path.find(".h265") != std::string::npos; + switch (video_type) { case VIDEO_H264: packager->copy_sps_pps_per_iframe = true; break; - case VIDEO_H264_MP4: - packager->copy_sps_pps_per_iframe = false; + case VIDEO_H264_CBR: + case VIDEO_H264_VBR: + packager->copy_sps_pps_per_iframe = is_raw_h264 ? true : false; break; - case VIDEO_H264_FLV: - packager->copy_sps_pps_per_iframe = false; - break; case VIDEO_H265: packager->copy_sps_pps_per_iframe = true; break; - case VIDEO_H265_MP4: - packager->copy_sps_pps_per_iframe = false; + case VIDEO_H265_CBR: + case VIDEO_H265_VBR: + packager->copy_sps_pps_per_iframe = is_raw_h265 ? true : false; break; default: err::check_raise(err::ERR_RUNTIME, "Unsupported video type!"); diff --git a/components/vision/port/maixcam/maix_webrtc_maixcam.cpp b/components/vision/port/maixcam/maix_webrtc_maixcam.cpp index 7adb6b56..7b4e667c 100644 --- a/components/vision/port/maixcam/maix_webrtc_maixcam.cpp +++ b/components/vision/port/maixcam/maix_webrtc_maixcam.cpp @@ -165,7 +165,6 @@ namespace maix::webrtc bool bind_audio_recorder; int encoder_bitrate; - int fps; int gop; MaixWebRTCServer *webrtc_server; @@ -257,20 +256,37 @@ namespace maix::webrtc param->status = WEBRTC_IDLE; } - WebRTC::WebRTC(std::string ip, int port, int fps, - webrtc::WebRTCStreamType stream_type, int bitrate, int gop, - std::string signaling_ip, int signaling_port, - const std::string &stun_server, - bool http_server) + static video::VideoType get_video_type( maix::webrtc::WebRTCStreamType stream, maix::webrtc::WebRTCRCType rc) + { + using stream_type = maix::webrtc::WebRTCStreamType; + using rc_type = maix::webrtc::WebRTCRCType; + + if (stream == stream_type::WEBRTC_STREAM_H264) { + if (rc == rc_type::WEBRTC_RC_CBR) { return video::VIDEO_H264_CBR; } + if (rc == rc_type::WEBRTC_RC_VBR) { return video::VIDEO_H264_VBR; } + } + + if (stream == stream_type::WEBRTC_STREAM_H265) { + if (rc == rc_type::WEBRTC_RC_CBR) { return video::VIDEO_H265_CBR; } + if (rc == rc_type::WEBRTC_RC_VBR) { return video::VIDEO_H265_VBR; } + } + + log::error("Unsupported video type stream=%d rc=%d", (int)stream, (int)rc); + return video::VIDEO_H264_CBR; + } + + WebRTC::WebRTC(std::string ip, int port, webrtc::WebRTCStreamType stream_type, + webrtc::WebRTCRCType rc_type, int bitrate, int gop, std::string signaling_ip, + int signaling_port, const std::string &stun_server, bool http_server) { this->_ip = ip.size() ? ip : "0.0.0.0"; this->_port = port; this->_signaling_ip = signaling_ip; this->_signaling_port = signaling_port; this->_stun_server = stun_server; - this->_fps = fps; this->_gop = gop; this->_stream_type = stream_type; + this->_rc_type = rc_type; this->_is_start = false; this->_video_thread = nullptr; this->_http_server = http_server; @@ -292,7 +308,6 @@ namespace maix::webrtc param->bind_camera = false; param->bind_audio_recorder = false; param->encoder_bitrate = bitrate; - param->fps = fps; param->gop = gop; param->webrtc_server = nullptr; @@ -353,17 +368,9 @@ namespace maix::webrtc param->encoder = nullptr; } - video::VideoType video_type; - if (this->_stream_type == maix::webrtc::WebRTCStreamType::WEBRTC_STREAM_H264) { - video_type = video::VIDEO_H264; - } else if (this->_stream_type == maix::webrtc::WebRTCStreamType::WEBRTC_STREAM_H265) { - video_type = video::VIDEO_H265; - } else { - log::error("Unsupported stream type: %d", this->_stream_type); - return err::ERR_ARGS; - } + video::VideoType video_type = get_video_type(this->_stream_type, this->_rc_type); - param->encoder = new video::Encoder("", param->camera->width(), param->camera->height(), image::Format::FMT_YVU420SP, video_type, param->fps, param->gop, param->encoder_bitrate); + param->encoder = new video::Encoder("", param->camera->width(), param->camera->height(), image::Format::FMT_YVU420SP, video_type, param->camera->fps(), param->gop, param->encoder_bitrate); err::check_null_raise(param->encoder, "Create video encoder failed!"); if (param->bind_audio_recorder) { @@ -376,7 +383,7 @@ namespace maix::webrtc } MaixWebRTCServerBuilder builder; - builder.set_ice_server("stun:stun.l.google.com:19302"); + builder.set_ice_server(this->_stun_server); if (this->_stream_type == maix::webrtc::WebRTCStreamType::WEBRTC_STREAM_H265) { builder.set_video_codec(VideoCodec::H265); @@ -541,6 +548,9 @@ namespace maix::webrtc if (!get_ip((char *)"wlan0", new_ip)) { ip_list.push_back("http://" + std::string(new_ip) + ":" + std::to_string(port)); } + if (!get_ip((char *)"tailscale0", new_ip)) { + ip_list.push_back("http://" + std::string(new_ip) + ":" + std::to_string(port)); + } } else { ip_list.push_back("http://" + ip + ":" + std::to_string(port)); } diff --git a/components/vision/port/maixcam2/maix_video_maixcam2.cpp b/components/vision/port/maixcam2/maix_video_maixcam2.cpp index af7dc8cb..c67a8df3 100644 --- a/components/vision/port/maixcam2/maix_video_maixcam2.cpp +++ b/components/vision/port/maixcam2/maix_video_maixcam2.cpp @@ -81,19 +81,46 @@ namespace maix::video } if (!strcmp(suffix, ".h264")) { - video_type = video::VIDEO_H264; + switch (type) { + case video::VIDEO_H264: // fall through + case video::VIDEO_H264_CBR: + video_type = video::VIDEO_H264_CBR; + break; + case video::VIDEO_H264_VBR: + video_type = video::VIDEO_H264_VBR; + break; + default: + err::check_raise(err::ERR_RUNTIME, "Unsupported video type!"); + break; + } } else if (!strcmp(suffix, ".h265")) { - video_type = video::VIDEO_H265; + switch (type) { + case video::VIDEO_H265: // fall through + case video::VIDEO_H265_CBR: + video_type = video::VIDEO_H265_CBR; + break; + case video::VIDEO_H265_VBR: + video_type = video::VIDEO_H265_VBR; + break; + default: + err::check_raise(err::ERR_RUNTIME, "Unsupported video type!"); + break; + } } else if (!strcmp(suffix, ".mp4")) { switch (type) { case video::VIDEO_H264: // fall through - case video::VIDEO_H264_MP4: // fall through - case video::VIDEO_H264_FLV: - video_type = video::VIDEO_H264_MP4; + case video::VIDEO_H264_CBR: + video_type = video::VIDEO_H264_CBR; + break; + case video::VIDEO_H264_VBR: + video_type = video::VIDEO_H264_VBR; break; case video::VIDEO_H265: // fall through - case video::VIDEO_H265_MP4: - video_type = video::VIDEO_H265_MP4; + case video::VIDEO_H265_CBR: + video_type = video::VIDEO_H265_CBR; + break; + case video::VIDEO_H265_VBR: + video_type = video::VIDEO_H265_VBR; break; default: err::check_raise(err::ERR_RUNTIME, "Unsupported video type!"); @@ -102,8 +129,11 @@ namespace maix::video } else if (!strcmp(suffix, ".flv")) { switch (type) { case video::VIDEO_H264: // fall through - case video::VIDEO_H264_FLV: - video_type = video::VIDEO_H264_FLV; + case video::VIDEO_H264_CBR: + video_type = video::VIDEO_H264_CBR; + break; + case video::VIDEO_H264_VBR: + video_type = video::VIDEO_H264_VBR; break; default: err::check_raise(err::ERR_RUNTIME, "Unsupported video type!"); @@ -120,12 +150,13 @@ namespace maix::video enum AVCodecID codec_id = AV_CODEC_ID_NONE; switch (video_type) { case VIDEO_H264: - case VIDEO_H264_MP4: - case VIDEO_H264_FLV: + case VIDEO_H264_CBR: + case VIDEO_H264_VBR: codec_id = AV_CODEC_ID_H264; break; case VIDEO_H265: - case VIDEO_H265_MP4: + case VIDEO_H265_CBR: + case VIDEO_H265_VBR: codec_id = AV_CODEC_ID_HEVC; break; default: @@ -313,21 +344,24 @@ namespace maix::video param->frame_index = 0; param->last_encode_ms = time::ticks_ms(); param->video_packet_list = new std::list; + + bool is_raw_h264 = _path.find(".h264") != std::string::npos; + bool is_raw_h265 = _path.find(".h265") != std::string::npos; + switch (video_type) { case VIDEO_H264: param->copy_sps_pps_per_iframe = true; break; - case VIDEO_H264_MP4: - param->copy_sps_pps_per_iframe = false; + case VIDEO_H264_CBR: + case VIDEO_H264_VBR: + param->copy_sps_pps_per_iframe = is_raw_h264 ? true : false; break; - case VIDEO_H264_FLV: - param->copy_sps_pps_per_iframe = false; - break; case VIDEO_H265: param->copy_sps_pps_per_iframe = true; break; - case VIDEO_H265_MP4: - param->copy_sps_pps_per_iframe = false; + case VIDEO_H265_CBR: + case VIDEO_H265_VBR: + param->copy_sps_pps_per_iframe = is_raw_h265 ? true : false; break; default: err::check_raise(err::ERR_RUNTIME, "Unsupported video type!"); @@ -336,12 +370,12 @@ namespace maix::video maixcam2::ax_venc_param_t cfg = {0}; switch (video_type) { case VIDEO_H264: - case VIDEO_H264_MP4: - case VIDEO_H264_FLV: + case VIDEO_H264_CBR: cfg.w = width; cfg.h = height; cfg.fmt = maixcam2::get_ax_fmt_from_maix(format); cfg.type = maixcam2::AX_VENC_TYPE_H264; + cfg.rc_mode = maixcam2::AX_VENC_RCMODE_CBR; cfg.h264.bitrate = bitrate / 1000; cfg.h264.input_fps = framerate; cfg.h264.output_fps = framerate; @@ -356,12 +390,31 @@ namespace maix::video cfg.h264.max_iprop = 40; cfg.h264.first_frame_start_qp = -1; break; + case VIDEO_H264_VBR: + cfg.w = width; + cfg.h = height; + cfg.fmt = maixcam2::get_ax_fmt_from_maix(format); + cfg.type = maixcam2::AX_VENC_TYPE_H264; + cfg.rc_mode = maixcam2::AX_VENC_RCMODE_VBR; + cfg.h264.bitrate = bitrate / 1000; + cfg.h264.input_fps = framerate; + cfg.h264.output_fps = framerate; + cfg.h264.gop = gop; + cfg.h264.intra_qp_delta = -2; + cfg.h264.de_breath_qp_delta = -2; + cfg.h264.min_qp = 31; + cfg.h264.max_qp = 46; + cfg.h264.min_iqp = 31; + cfg.h264.max_iqp = 46; + cfg.h264.first_frame_start_qp = -1; + break; case VIDEO_H265: - case VIDEO_H265_MP4: + case VIDEO_H265_CBR: cfg.w = width; cfg.h = height; cfg.fmt = maixcam2::get_ax_fmt_from_maix(format); cfg.type = maixcam2::AX_VENC_TYPE_H265; + cfg.rc_mode = maixcam2::AX_VENC_RCMODE_CBR; cfg.h265.bitrate = bitrate / 1000; cfg.h265.input_fps = framerate; cfg.h265.output_fps = framerate; @@ -376,10 +429,25 @@ namespace maix::video cfg.h265.max_iprop = 40; cfg.h265.first_frame_start_qp = -1; cfg.h265.qp_delta_rgn = 10; - cfg.h265.qp_map_type = AX_VENC_QPMAP_QP_DISABLE; - cfg.h265.qp_map_blk_type = AX_VENC_QPMAP_BLOCK_DISABLE; - cfg.h265.qp_map_block_unit = AX_VENC_QPMAP_BLOCK_UNIT_64x64; - cfg.h265.ctb_rc_mode = AX_VENC_RC_CTBRC_DISABLE; + break; + case VIDEO_H265_VBR: + cfg.w = width; + cfg.h = height; + cfg.fmt = maixcam2::get_ax_fmt_from_maix(format); + cfg.rc_mode = maixcam2::AX_VENC_RCMODE_VBR; + cfg.type = maixcam2::AX_VENC_TYPE_H265; + cfg.h265.bitrate = bitrate / 1000; + cfg.h265.input_fps = framerate; + cfg.h265.output_fps = framerate; + cfg.h265.gop = gop; + cfg.h265.intra_qp_delta = -2; + cfg.h265.de_breath_qp_delta = -2; + cfg.h265.min_qp = 31; + cfg.h265.max_qp = 46; + cfg.h265.min_iqp = 31; + cfg.h265.max_iqp = 46; + cfg.h265.first_frame_start_qp = -1; + cfg.h265.qp_delta_rgn = 10; break; default: err::check_raise(err::ERR_RUNTIME, "unsupport stream type!"); @@ -2050,10 +2118,12 @@ __vdec_exit: ::close(this->_fd); switch (this->_video_type) { - case VIDEO_ENC_H265_CBR: + case VIDEO_H265_CBR: + case VIDEO_H265_VBR: // do nothing break; - case VIDEO_ENC_MP4_CBR: + case VIDEO_H264_CBR: + case VIDEO_H264_VBR: { char cmd[128]; snprintf(cmd, sizeof(cmd), "ffmpeg -loglevel quiet -i %s -c:v copy -c:a copy %s -y", this->_tmp_path.c_str(), this->_path.c_str()); diff --git a/components/vision/port/maixcam2/maix_webrtc_maixcam2.cpp b/components/vision/port/maixcam2/maix_webrtc_maixcam2.cpp index 2655d5e5..cd734d10 100644 --- a/components/vision/port/maixcam2/maix_webrtc_maixcam2.cpp +++ b/components/vision/port/maixcam2/maix_webrtc_maixcam2.cpp @@ -66,7 +66,6 @@ namespace maix::webrtc bool bind_audio_recorder; int encoder_bitrate; - int fps; int gop; MaixWebRTCServer *webrtc_server; @@ -151,20 +150,37 @@ namespace maix::webrtc param->status = WEBRTC_IDLE; } - WebRTC::WebRTC(std::string ip, int port, int fps, - webrtc::WebRTCStreamType stream_type, int bitrate, int gop, - std::string signaling_ip, int signaling_port, - const std::string &stun_server, - bool http_server) + static video::VideoType get_video_type( maix::webrtc::WebRTCStreamType stream, maix::webrtc::WebRTCRCType rc) + { + using stream_type = maix::webrtc::WebRTCStreamType; + using rc_type = maix::webrtc::WebRTCRCType; + + if (stream == stream_type::WEBRTC_STREAM_H264) { + if (rc == rc_type::WEBRTC_RC_CBR) { return video::VIDEO_H264_CBR; } + if (rc == rc_type::WEBRTC_RC_VBR) { return video::VIDEO_H264_VBR; } + } + + if (stream == stream_type::WEBRTC_STREAM_H265) { + if (rc == rc_type::WEBRTC_RC_CBR) { return video::VIDEO_H265_CBR; } + if (rc == rc_type::WEBRTC_RC_VBR) { return video::VIDEO_H265_VBR; } + } + + log::error("Unsupported video type stream=%d rc=%d", (int)stream, (int)rc); + return video::VIDEO_H264_CBR; + } + + WebRTC::WebRTC(std::string ip, int port, webrtc::WebRTCStreamType stream_type, + webrtc::WebRTCRCType rc_type, int bitrate, int gop, std::string signaling_ip, + int signaling_port, const std::string &stun_server, bool http_server) { this->_ip = ip.size() ? ip : "0.0.0.0"; this->_port = port; this->_signaling_ip = signaling_ip; this->_signaling_port = signaling_port; this->_stun_server = stun_server; - this->_fps = fps; this->_gop = gop; this->_stream_type = stream_type; + this->_rc_type = rc_type; this->_is_start = false; this->_video_thread = nullptr; this->_http_server = http_server; @@ -186,7 +202,6 @@ namespace maix::webrtc param->bind_camera = false; param->bind_audio_recorder = false; param->encoder_bitrate = bitrate; - param->fps = fps; param->gop = gop; param->webrtc_server = nullptr; @@ -247,17 +262,9 @@ namespace maix::webrtc param->encoder = nullptr; } - video::VideoType video_type; - if (this->_stream_type == maix::webrtc::WebRTCStreamType::WEBRTC_STREAM_H264) { - video_type = video::VIDEO_H264; - } else if (this->_stream_type == maix::webrtc::WebRTCStreamType::WEBRTC_STREAM_H265) { - video_type = video::VIDEO_H265; - } else { - log::error("Unsupported stream type: %d", this->_stream_type); - return err::ERR_ARGS; - } + video::VideoType video_type = get_video_type(this->_stream_type, this->_rc_type); - param->encoder = new video::Encoder("", param->camera->width(), param->camera->height(), image::Format::FMT_YVU420SP, video_type, param->fps, param->gop, param->encoder_bitrate); + param->encoder = new video::Encoder("", param->camera->width(), param->camera->height(), image::Format::FMT_YVU420SP, video_type, param->camera->fps(), param->gop, param->encoder_bitrate); err::check_null_raise(param->encoder, "Create video encoder failed!"); if (param->bind_audio_recorder) { @@ -270,7 +277,7 @@ namespace maix::webrtc } MaixWebRTCServerBuilder builder; - builder.set_ice_server("stun:stun.l.google.com:19302"); + builder.set_ice_server(this->_stun_server); if (this->_stream_type == maix::webrtc::WebRTCStreamType::WEBRTC_STREAM_H265) { builder.set_video_codec(VideoCodec::H265);