* support record audio and save to mp4

components/vision/include/maix_video.hpp

@@ -628,10 +628,11 @@ namespace maix::video
          * Encode image.
          * @param img the image will be encode.
          * if the img is NULL, this function will try to get image from camera, you must use bind_camera() function to bind the camera.
+         * @param pcm the pcm data will be encode.
          * @return encode result
          * @maixpy maix.video.Encoder.encode
         */
-        video::Frame *encode(image::Image *img = maix::video::Encoder::NoneImage);
+        video::Frame *encode(image::Image *img = maix::video::Encoder::NoneImage, Bytes *pcm = NULL);
 
         /**
          * Capture image

components/vision/port/linux/maix_video_linux.cpp

@@ -44,8 +44,9 @@ namespace maix::video
         return err::ERR_NOT_IMPL;
     }
 
-    video::Frame *Encoder::encode(image::Image *img) {
+    video::Frame *Encoder::encode(image::Image *img, Bytes *pcm) {
         (void)img;
+        (void)pcm;
         throw err::Exception(err::ERR_NOT_IMPL);
         return nullptr;
     }

components/vision/port/maixcam/maix_video_mmf.cpp

@@ -195,6 +195,19 @@ namespace maix::video
         video::VideoType video_type;
         int venc_ch;
         PAYLOAD_TYPE_E venc_type;
+
+        AVStream *audio_stream = NULL;
+        AVCodecContext *audio_codec_ctx = NULL;
+        AVCodec *audio_codec = NULL;
+        AVFrame *audio_frame = NULL;
+        SwrContext *swr_ctx = NULL;
+        AVPacket *audio_packet = NULL;
+        uint64_t audio_last_pts = 0;
+        std::list<Bytes *> *pcm_list;
+        int audio_sample_rate;
+        int audio_channels;
+        int audio_bitrate;
+        enum AVSampleFormat audio_format;
     } encoder_param_t;
 
     Encoder::Encoder(std::string path, int width, int height, image::Format format, VideoType type, int framerate, int gop, int bitrate, int time_base, bool capture, bool block) {
@@ -273,6 +286,8 @@ namespace maix::video
                 log::error("Count not open file: %s", _path.c_str());
                 err::check_raise(err::ERR_RUNTIME, "Could not open file");
             }
+
+            /* video init */
             AVStream *outputStream = avformat_new_stream(outputFormatContext, NULL);
             err::check_null_raise(outputStream, "create new stream failed");
 
@@ -291,8 +306,6 @@ namespace maix::video
                 }
             }
 
-            err::check_bool_raise(avformat_write_header(outputFormatContext, NULL) >= 0, "avformat_write_header failed!");
-
             AVPacket *pPacket = av_packet_alloc();
             err::check_null_raise(pPacket, "malloc failed!");
 
@@ -323,9 +336,64 @@ namespace maix::video
                 err::check_raise(err::ERR_RUNTIME, "mmf venc init failed!");
             }
 
+            /* audio init */
+            AVStream *audio_stream = NULL;
+            AVCodecContext *audio_codec_ctx = NULL;
+            AVCodec *audio_codec = NULL;
+            AVFrame *audio_frame = NULL;
+            SwrContext *swr_ctx = NULL;
+            AVPacket *audio_packet = NULL;
+
+            audio_packet = av_packet_alloc();
+            err::check_null_raise(audio_packet, "av_packet_alloc");
+            audio_packet->data = NULL;
+            audio_packet->size = 0;
+
+            int sample_rate = 48000;
+            int channels = 1;
+            int bitrate = 128000;
+            enum AVSampleFormat format = AV_SAMPLE_FMT_S16;
+            err::check_null_raise(audio_codec = avcodec_find_encoder(AV_CODEC_ID_AAC), "Could not find aac encoder");
+            err::check_null_raise(audio_stream = avformat_new_stream(outputFormatContext, NULL), "Could not allocate stream");
+            err::check_null_raise(audio_codec_ctx = avcodec_alloc_context3(audio_codec), "Could not allocate audio codec context");
+            audio_codec_ctx->codec_id = AV_CODEC_ID_AAC;
+            audio_codec_ctx->codec_type = AVMEDIA_TYPE_AUDIO;
+            audio_codec_ctx->sample_rate = sample_rate;
+            audio_codec_ctx->channels = channels;
+            audio_codec_ctx->channel_layout = av_get_default_channel_layout(audio_codec_ctx->channels);
+            audio_codec_ctx->sample_fmt = AV_SAMPLE_FMT_FLTP;  // AAC编码需要浮点格式
+            audio_codec_ctx->time_base = (AVRational){1, sample_rate};
+            audio_codec_ctx->bit_rate = bitrate;
+            audio_stream->time_base = audio_codec_ctx->time_base;
+            err::check_bool_raise(avcodec_parameters_from_context(audio_stream->codecpar, audio_codec_ctx) >= 0, "avcodec_parameters_to_context");
+            err::check_bool_raise(avcodec_open2(audio_codec_ctx, audio_codec, NULL) >= 0, "audio_codec open failed");
+log::info("audio stream index:%d video index:%d outputFormatContext outputFormatContext->nb_streams:%d timebase:%d/%d",
+audio_stream->index, outputStream->index, outputFormatContext->nb_streams, audio_stream->time_base.num, audio_stream->time_base.den);
+            swr_ctx = swr_alloc();
+            av_opt_set_int(swr_ctx, "in_channel_layout", audio_codec_ctx->channel_layout, 0);
+            av_opt_set_int(swr_ctx, "out_channel_layout", audio_codec_ctx->channel_layout, 0);
+            av_opt_set_int(swr_ctx, "in_sample_rate", audio_codec_ctx->sample_rate, 0);
+            av_opt_set_int(swr_ctx, "out_sample_rate", audio_codec_ctx->sample_rate, 0);
+            av_opt_set_sample_fmt(swr_ctx, "in_sample_fmt", format, 0);
+            av_opt_set_sample_fmt(swr_ctx, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
+            swr_init(swr_ctx);
+
+            int frame_size = audio_codec_ctx->frame_size;
+            audio_frame = av_frame_alloc();
+            audio_frame->nb_samples = frame_size;
+            audio_frame->channel_layout = audio_codec_ctx->channel_layout;
+            audio_frame->format = AV_SAMPLE_FMT_FLTP;
+            audio_frame->sample_rate = audio_codec_ctx->sample_rate;
+            av_frame_get_buffer(audio_frame, 0);
+
+            err::check_bool_raise(avformat_write_header(outputFormatContext, NULL) >= 0, "avformat_write_header failed!");
+
+            /* param init */
             encoder_param_t *param = (encoder_param_t *)_param;
             param->outputFormatContext = outputFormatContext;
             param->outputStream = outputStream;
+
+            // video init
             param->pPacket = pPacket;
             param->video_type = video_type;
             param->venc_ch = MMF_VENC_CHN;
@@ -352,6 +420,20 @@ namespace maix::video
                 default:
                     err::check_raise(err::ERR_RUNTIME, "Unsupported video type!");
             }
+
+            // audio init
+            param->audio_stream = audio_stream;
+            param->audio_codec_ctx = audio_codec_ctx;
+            param->audio_codec = audio_codec;
+            param->audio_frame = audio_frame;
+            param->swr_ctx = swr_ctx;
+            param->audio_packet = audio_packet;
+            param->audio_last_pts = 0;
+            param->pcm_list = new std::list<Bytes *>;
+            param->audio_sample_rate = 48000;
+            param->audio_channels = 1;
+            param->audio_bitrate = 128000;
+            param->audio_format = AV_SAMPLE_FMT_S16;
         }
     }
 
@@ -385,6 +467,17 @@ namespace maix::video
                 mmf_del_venc_channel(MMF_VENC_CHN);
                 mmf_deinit_v2(false);
                 av_write_trailer(param->outputFormatContext);
+
+                for (auto it = param->pcm_list->begin(); it != param->pcm_list->end(); ++it) {
+                    Bytes *pcm = *it;
+                    delete pcm;
+                    param->pcm_list->erase(it);
+                }
+                delete param->pcm_list;
+                av_frame_free(&param->audio_frame);
+                swr_free(&param->swr_ctx);
+                avcodec_free_context(&param->audio_codec_ctx);
+
                 avformat_close_input(&param->outputFormatContext);
                 if (param->outputFormatContext && !(param->outputFormatContext->oformat->flags & AVFMT_NOFILE)) {
                     avio_closep(&param->outputFormatContext->pb);
@@ -392,6 +485,7 @@ namespace maix::video
                 avformat_free_context(param->outputFormatContext);
                 av_packet_unref(param->pPacket);
                 av_packet_free(&param->pPacket);
+
                 free(_param);
                 _param = NULL;
             }
@@ -417,7 +511,7 @@ namespace maix::video
         return err;
     }
 
-    video::Frame *Encoder::encode(image::Image *img) {
+    video::Frame *Encoder::encode(image::Image *img, Bytes *pcm) {
 #if 1
         uint8_t *stream_buffer = NULL;
         int stream_size = 0;
@@ -937,6 +1031,9 @@ namespace maix::video
                             }
                             pPacket->data = stream_buffer;
                             pPacket->size = copy_length;
+
+                            // std::vector<int> timebase = {param->outputStream->time_base.num, param->outputStream->time_base.den};
+                            // log::info("[VIDEO] pts:%d duration:%d time:%.2f", pPacket->pts, pPacket->duration, timebase_to_ms(timebase, pPacket->pts) / 1000);
                             err::check_bool_raise(av_interleaved_write_frame(param->outputFormatContext, pPacket) >= 0, "av_interleaved_write_frame failed!");
                         }
                     }
@@ -949,6 +1046,91 @@ namespace maix::video
                     goto _exit;
                 }
 
+                // audio process
+                if (pcm && pcm->data_len > 0) {
+                    // uint64_t t = time::ticks_ms();
+                    std::list<Bytes *> *pcm_list = param->pcm_list;
+                    AVFrame *audio_frame = param->audio_frame;
+                    AVStream *audio_stream = param->audio_stream;
+                    AVCodecContext *audio_codec_ctx = param->audio_codec_ctx;
+                    SwrContext *swr_ctx = param->swr_ctx;
+                    AVFormatContext *outputFormatContext = param->outputFormatContext;
+                    AVPacket *audio_packet = param->audio_packet;
+                    size_t buffer_size = av_samples_get_buffer_size(NULL, 1, audio_frame->nb_samples, param->audio_format, 1);
+                    size_t pcm_remain_len = pcm->data_len;
+                    // fill last pcm to buffer_size
+                    Bytes *last_pcm = pcm_list->back();
+                    if (last_pcm && last_pcm->data_len < buffer_size) {
+                        int temp_size = pcm_remain_len + last_pcm->data_len >= buffer_size ? buffer_size : pcm_remain_len + last_pcm->data_len;
+                        uint8_t *temp = (uint8_t *)malloc(temp_size);
+                        err::check_null_raise(temp, "malloc failed!");
+                        memcpy(temp, last_pcm->data, last_pcm->data_len);
+                        if (pcm_remain_len + last_pcm->data_len < buffer_size) {
+                            memcpy(temp + last_pcm->data_len, pcm->data, pcm_remain_len);
+                            pcm_remain_len = 0;
+                        } else {
+                            memcpy(temp + last_pcm->data_len, pcm->data, buffer_size - last_pcm->data_len);
+                            pcm_remain_len -= (buffer_size - last_pcm->data_len);
+                        }
+
+                        Bytes *new_pcm = new Bytes(temp, temp_size, true, false);
+                        pcm_list->pop_back();
+                        delete last_pcm;
+                        pcm_list->push_back(new_pcm);
+                    }
+
+                    // fill other pcm
+                    while (pcm_remain_len > 0) {
+                        int temp_size = pcm_remain_len >= buffer_size ? buffer_size : pcm_remain_len;
+                        uint8_t *temp = (uint8_t *)malloc(temp_size);
+                        err::check_null_raise(temp, "malloc failed!");
+                        memcpy(temp, pcm->data + pcm->data_len - pcm_remain_len, temp_size);
+                        pcm_remain_len -= temp_size;
+
+                        Bytes *new_pcm = new Bytes(temp, temp_size, true, false);
+                        pcm_list->push_back(new_pcm);
+                    }
+
+                    // audio process
+                    while (pcm_list->size() > 0) {
+                        Bytes *pcm = pcm_list->front();
+                        if (pcm) {
+                            if (pcm->data_len == buffer_size) {
+                                // save to mp4
+                                // log::info("pcm data:%p size:%d list_size:%d", pcm->data, pcm->data_len, pcm_list->size());
+                                const uint8_t *in[] = {pcm->data};
+                                uint8_t *out[] = {audio_frame->data[0]};
+                                swr_convert(swr_ctx, out, audio_codec_ctx->frame_size, in, audio_codec_ctx->frame_size);
+                                audio_frame->pts = param->audio_last_pts;
+                                if (avcodec_send_frame(audio_codec_ctx, audio_frame) < 0) {
+                                    printf("Error sending audio_frame to encoder.\n");
+                                    break;
+                                }
+
+                                while (avcodec_receive_packet(audio_codec_ctx, audio_packet) == 0) {
+                                    audio_packet->stream_index = audio_stream->index;
+                                    audio_packet->pts = param->audio_last_pts;
+                                    audio_packet->dts = audio_packet->pts;
+                                    audio_packet->duration = audio_codec_ctx->frame_size;
+                                    param->audio_last_pts = audio_packet->pts + audio_packet->duration;
+
+                                    // std::vector<int> timebase = {audio_stream->time_base.num, audio_stream->time_base.den};
+                                    // log::info("[AUDIO] pts:%d duration:%d timebase:%d/%d time:%.2f", audio_packet->pts, audio_packet->duration, audio_stream->time_base.num, audio_stream->time_base.den, timebase_to_ms(timebase, audio_packet->pts) / 1000);
+                                    av_interleaved_write_frame(outputFormatContext, audio_packet);
+                                    av_packet_unref(audio_packet);
+                                }
+                                pcm_list->pop_front();
+                                delete pcm;
+                            } else {
+                                break;
+                            }
+                        } else {
+                            err::check_raise(err::ERR_RUNTIME, "pcm data error");
+                        }
+                    }
+                    // log::info("pcm process time:%d", time::ticks_ms() - t);
+                }
+
                 if (!_block) {
                     if (use_input_img) {
                         while ((time::ticks_ms() - param->last_encode_ms) * _framerate < 1000) {
@@ -2006,8 +2188,6 @@ _retry:
         decoder_param_t *param = (decoder_param_t *)_param;
         AVPacket *pPacket = param->pPacket;
         AVFormatContext *pFormatContext = param->pFormatContext;
-        AVBSFContext * bsfc = param->bsfc;
-        int video_stream_index = param->video_stream_index;
         int audio_stream_index = param->audio_stream_index;
         AVCodecContext *audio_codec_ctx = param->audio_codec_ctx;
         AVFrame *audio_frame = param->audio_frame;
@@ -2015,11 +2195,7 @@ _retry:
         int resample_sample_rate = param->resample_sample_rate;
         enum AVSampleFormat resample_format = param->resample_sample_format;
         SwrContext *swr_ctx = param->swr_ctx;
-        image::Image *img = NULL;
         video::Context *context = NULL;
-        uint64_t last_pts = 0;
-        uint64_t curr_pts = param->next_pts;
-        int frame_duration = pFormatContext->streams[video_stream_index]->time_base.den / pFormatContext->streams[video_stream_index]->time_base.num / _fps;
         while (av_read_frame(pFormatContext, pPacket) >= 0) {
             if (pPacket->stream_index == audio_stream_index) {
                 if (avcodec_send_packet(audio_codec_ctx, pPacket) >= 0) {
@@ -2079,11 +2255,10 @@ _retry:
         image::Image *img = NULL;
         video::Context *context = NULL;
         uint64_t last_pts = 0;
-        uint64_t curr_pts = param->next_pts;
         int frame_duration = pFormatContext->streams[video_stream_index]->time_base.den / pFormatContext->streams[video_stream_index]->time_base.num / _fps;
         bool is_video = false;
         bool is_audio = false;
-_retry:
+
         while (av_read_frame(pFormatContext, pPacket) >= 0) {
             log::info("[READ] audio/video:%d pts:%d pts_ms:%.2f ms",
             pPacket->stream_index, pPacket->pts, pPacket->pts / ((float)pFormatContext->streams[pPacket->stream_index]->time_base.den/pFormatContext->streams[pPacket->stream_index]->time_base.num));

examples/encode_demo/main/src/main.cpp

@@ -6,18 +6,40 @@
 #include "maix_display.hpp"
 #include "maix_video.hpp"
 #include "maix_camera.hpp"
-
+#include "list"
 using namespace maix;
 
 extern "C" {
 #include <libavformat/avformat.h>
 #include <libavcodec/avcodec.h>
+#include <stdio.h>
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+#include <libavutil/opt.h>
+#include <libavutil/samplefmt.h>
+#include <libavutil/channel_layout.h>
+#include <libswresample/swresample.h>
+}
+
+static double timebase_to_ms(std::vector<int> timebase, uint64_t value) {
+    return value * 1000 / ((double)timebase[1] / timebase[0]);
 }
 
 static int h264_to_mp4(int argc, char *argv[]) {
     const char *input_filename = "output.h264";
     const char *output_filename = "output2.mp4";
 
+    AVStream *audio_stream = NULL;
+    AVCodecContext *audio_codec_ctx = NULL;
+    AVCodec *audio_codec = NULL;
+    AVFrame *audio_frame = NULL;
+    SwrContext *swr_ctx = NULL;
+    AVPacket *audio_packet = NULL;
+    audio_packet = av_packet_alloc();
+    err::check_null_raise(audio_packet, "av_packet_alloc");
+    audio_packet->data = NULL;
+    audio_packet->size = 0;
+
     AVFormatContext *input_format_ctx = NULL;
     AVFormatContext *output_format_ctx = NULL;
     AVPacket packet;
@@ -57,12 +79,52 @@ static int h264_to_mp4(int argc, char *argv[]) {
         // 复制编码器参数
         ret = avcodec_parameters_copy(out_stream->codecpar, in_stream->codecpar);
         if (ret < 0) {
-            fprintf(stderr, "Failed to copy codec parameters\n");
+            fprintf(stderr, "Failed to copy audio_codec parameters\n");
             return -1;
         }
         out_stream->codecpar->codec_tag = 0;
     }
 
+    enum AVSampleFormat format = AV_SAMPLE_FMT_S16;
+    int sample_rate = 48000;
+    int channels = 1;
+    int bitrate = 128000;
+    {
+        err::check_null_raise(audio_codec = avcodec_find_encoder(AV_CODEC_ID_AAC), "Could not find aac encoder");
+        err::check_null_raise(audio_stream = avformat_new_stream(output_format_ctx, NULL), "Could not allocate stream");
+        err::check_null_raise(audio_codec_ctx = avcodec_alloc_context3(audio_codec), "Could not allocate audio codec context");
+        audio_codec_ctx->codec_id = AV_CODEC_ID_AAC;
+        audio_codec_ctx->codec_type = AVMEDIA_TYPE_AUDIO;
+        audio_codec_ctx->sample_rate = sample_rate;
+        audio_codec_ctx->channels = channels;
+        audio_codec_ctx->channel_layout = av_get_default_channel_layout(audio_codec_ctx->channels);
+        audio_codec_ctx->sample_fmt = AV_SAMPLE_FMT_FLTP;  // AAC编码需要浮点格式
+        audio_codec_ctx->bit_rate = bitrate;
+        audio_stream->time_base = (AVRational){1, sample_rate};
+        err::check_bool_raise(avcodec_parameters_from_context(audio_stream->codecpar, audio_codec_ctx) >= 0, "avcodec_parameters_to_context");
+        err::check_bool_raise(avcodec_open2(audio_codec_ctx, audio_codec, NULL) >= 0, "audio_codec open failed");
+
+        swr_ctx = swr_alloc();
+        av_opt_set_int(swr_ctx, "in_channel_layout", audio_codec_ctx->channel_layout, 0);
+        av_opt_set_int(swr_ctx, "out_channel_layout", audio_codec_ctx->channel_layout, 0);
+        av_opt_set_int(swr_ctx, "in_sample_rate", audio_codec_ctx->sample_rate, 0);
+        av_opt_set_int(swr_ctx, "out_sample_rate", audio_codec_ctx->sample_rate, 0);
+        av_opt_set_sample_fmt(swr_ctx, "in_sample_fmt", format, 0);
+        av_opt_set_sample_fmt(swr_ctx, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
+        swr_init(swr_ctx);
+    }
+
+    int frame_size = audio_codec_ctx->frame_size;
+    size_t buffer_size = av_samples_get_buffer_size(NULL, 1, frame_size, format, 1);
+    audio_frame = av_frame_alloc();
+    audio_frame->nb_samples = frame_size;
+    audio_frame->channel_layout = audio_codec_ctx->channel_layout;
+    audio_frame->format = AV_SAMPLE_FMT_FLTP;
+    audio_frame->sample_rate = audio_codec_ctx->sample_rate;
+    av_frame_get_buffer(audio_frame, 0);
+
+    int last_pts = 0;
+
     // 打开输出文件
     if (!(output_format_ctx->oformat->flags & AVFMT_NOFILE)) {
         ret = avio_open(&output_format_ctx->pb, output_filename, AVIO_FLAG_WRITE);
@@ -81,7 +143,99 @@ static int h264_to_mp4(int argc, char *argv[]) {
 
     int count = 0;
     // 读取输入数据包并写入输出文件
-    while (av_read_frame(input_format_ctx, &packet) >= 0) {
+
+    audio::Recorder *r = new audio::Recorder();
+    Bytes *pcm = NULL;
+    std::list<Bytes *> *pcm_list = new std::list<Bytes *>;
+
+    while (av_read_frame(input_format_ctx, &packet) >= 0 && !app::need_exit()) {
+{
+        uint64_t t = time::ticks_ms();
+        pcm = r->record();
+        if (pcm) {
+            if (pcm->data_len > 0) {
+                log::info("record use %lld ms", time::ticks_ms() - t);
+                t = time::ticks_ms();
+                size_t pcm_remain_len = pcm->data_len;
+                // fill last pcm to buffer_size
+                Bytes *last_pcm = pcm_list->back();
+                if (last_pcm && last_pcm->data_len < buffer_size) {
+                    int temp_size = pcm_remain_len + last_pcm->data_len >= buffer_size ? buffer_size : pcm_remain_len + last_pcm->data_len;
+                    uint8_t *temp = (uint8_t *)malloc(temp_size);
+                    err::check_null_raise(temp, "malloc failed!");
+                    memcpy(temp, last_pcm->data, last_pcm->data_len);
+                    if (pcm_remain_len + last_pcm->data_len < buffer_size) {
+                        memcpy(temp + last_pcm->data_len, pcm->data, pcm_remain_len);
+                        pcm_remain_len = 0;
+                    } else {
+                        memcpy(temp + last_pcm->data_len, pcm->data, buffer_size - last_pcm->data_len);
+                        pcm_remain_len -= (buffer_size - last_pcm->data_len);
+                    }
+
+                    Bytes *new_pcm = new Bytes(temp, temp_size, true, false);
+                    pcm_list->pop_back();
+                    delete last_pcm;
+                    pcm_list->push_back(new_pcm);
+                }
+
+                // fill other pcm
+                while (pcm_remain_len > 0) {
+                    int temp_size = pcm_remain_len >= buffer_size ? buffer_size : pcm_remain_len;
+                    uint8_t *temp = (uint8_t *)malloc(temp_size);
+                    err::check_null_raise(temp, "malloc failed!");
+                    memcpy(temp, pcm->data + pcm->data_len - pcm_remain_len, temp_size);
+                    pcm_remain_len -= temp_size;
+
+                    Bytes *new_pcm = new Bytes(temp, temp_size, true, false);
+                    pcm_list->push_back(new_pcm);
+                }
+
+                // audio process
+                while (pcm_list->size() > 0) {
+                    Bytes *pcm = pcm_list->front();
+                    if (pcm) {
+                        if (pcm->data_len == buffer_size) {
+                            // save to mp4
+                            // log::info("pcm data:%p size:%d list_size:%d", pcm->data, pcm->data_len, pcm_list->size());
+                            const uint8_t *in[] = {pcm->data};
+                            uint8_t *out[] = {audio_frame->data[0]};
+                            swr_convert(swr_ctx, out, frame_size, in, frame_size);
+                            audio_frame->pts = last_pts;
+                            if (avcodec_send_frame(audio_codec_ctx, audio_frame) < 0) {
+                                printf("Error sending audio_frame to encoder.\n");
+                                break;
+                            }
+
+                            while (avcodec_receive_packet(audio_codec_ctx, audio_packet) == 0) {
+                                audio_packet->stream_index = audio_stream->index;
+                                audio_packet->pts = last_pts;
+                                audio_packet->dts = audio_packet->pts;
+                                audio_packet->duration = audio_codec_ctx->frame_size;
+                                last_pts = audio_packet->pts + audio_packet->duration;
+                                // log::info("audio_frame->pts:%d packet: pts:%d dts:%d duration:%d", audio_frame->pts, audio_packet->pts, audio_packet->dts, audio_packet->duration);
+
+                                std::vector<int> timebase = {audio_stream->time_base.num, audio_stream->time_base.den};
+                                log::info("[AUDIO] id:%d pts:%d dts:%d duration:%d timebase:%d/%d time:%.2f", audio_packet->stream_index, audio_packet->pts, audio_packet->dts, audio_packet->duration, audio_stream->time_base.num, audio_stream->time_base.den, timebase_to_ms(timebase, audio_packet->pts) / 1000);
+                                av_interleaved_write_frame(output_format_ctx, audio_packet);
+                                av_packet_unref(audio_packet);
+                            }
+
+                            pcm_list->pop_front();
+                            delete pcm;
+                        } else {
+                            break;
+                        }
+                    } else {
+                        err::check_raise(err::ERR_RUNTIME, "pcm data error");
+                    }
+                }
+                // log::info("save to mp4 use %lld ms", time::ticks_ms() - t);
+            }
+            delete pcm;
+        }
+        time::sleep_ms(33);
+}
+
         AVStream *in_stream = input_format_ctx->streams[packet.stream_index];
         AVStream *out_stream = output_format_ctx->streams[packet.stream_index];
         count ++;
@@ -106,6 +260,7 @@ static int h264_to_mp4(int argc, char *argv[]) {
         }
 
         // 将时间戳从输入流转换为输出流
+        packet.stream_index = out_stream->index;
         packet.pts = av_rescale_q_rnd(packet.pts, in_stream->time_base, out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));
         packet.dts = av_rescale_q_rnd(packet.dts, in_stream->time_base, out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));
         packet.duration = av_rescale_q(packet.duration, in_stream->time_base, out_stream->time_base);
@@ -113,12 +268,15 @@ static int h264_to_mp4(int argc, char *argv[]) {
         packet.dts = count * packet.duration;
         packet.pos = -1;
 
+        std::vector<int> timebase = {out_stream->time_base.num, out_stream->time_base.den};
+        log::info("[VIDEO] id:%d pts:%d dts:%d duration:%d time:%.2f", packet.stream_index, packet.pts, packet.dts, packet.duration, timebase_to_ms(timebase, packet.pts) / 1000);
         // 写入包到输出文件
         ret = av_interleaved_write_frame(output_format_ctx, &packet);
         if (ret < 0) {
             fprintf(stderr, "Error muxing packet\n");
             break;
         }
+
         av_packet_unref(&packet);
     }
 
@@ -178,6 +336,7 @@ int _main(int argc, char* argv[])
         int height = 480;
         image::Format format = image::Format::FMT_YVU420SP;
         video::VideoType type = video::VIDEO_H264;
+        audio::Recorder r = audio::Recorder();
         int framerate = 30;
         int gop = 50;
         int bitrate = 3000 * 1000;
@@ -203,9 +362,9 @@ int _main(int argc, char* argv[])
 
         while(!app::need_exit()) {
             image::Image *img = cam.read();
-            video::Frame *frame = e.encode(img);
-            // printf("frame data:%p size:%ld pts:%ld dts:%ld\r\n",
-            //     frame->data(), frame->size(), frame->get_pts(), frame->get_dts());
+            Bytes *pcm = r.record();
+            video::Frame *frame = e.encode(img, pcm);
+            delete pcm;
             delete frame;
 
             disp.show(*img);
@@ -232,17 +391,17 @@ int _main(int argc, char* argv[])
         video::Encoder e = video::Encoder(path, width, height, format, type, framerate, gop, bitrate, time_base, capture);
         camera::Camera cam = camera::Camera(width, height, format);
         display::Display disp = display::Display();
+        audio::Recorder r = audio::Recorder();
         e.bind_camera(&cam);
 
         while(!app::need_exit()) {
-            video::Frame *frame = e.encode();
+            Bytes *pcm = r.record();
+            video::Frame *frame = e.encode(NULL, pcm);
             image::Image *img = e.capture();
-            printf("frame data:%p size:%ld pts:%ld dts:%ld\r\n",
-                frame->data(), frame->size(), frame->get_pts(), frame->get_dts());
-
             disp.show(*img);
             delete frame;
             delete img;
+            delete pcm;
         }
         break;
     }
@@ -279,10 +438,10 @@ int _main(int argc, char* argv[])
 
             if (time::ticks_ms() - last_ms >= delay_record) {
                 last_ms = time::ticks_ms();
-                video::Frame *frame = e.encode(img);
-                printf("frame data:%p size:%ld pts:%ld dts:%ld\r\n",
-                    frame->data(), frame->size(), frame->get_pts(), frame->get_dts());
-                delete frame;
+                video::Frame *audio_frame = e.encode(img);
+                printf("audio_frame data:%p size:%ld pts:%ld dts:%ld\r\n",
+                    audio_frame->data(), audio_frame->size(), audio_frame->get_pts(), audio_frame->get_dts());
+                delete audio_frame;
                 continue;
             }