前言以前的大作业根据rknn_model_zoo和easy eai示例代码修改缝合仅供参考后来我试着模块化一些方便看但因为核心代码都是直接用的示例代码所以有些模块还是耦合compositor和display,有些模块干脆没有发挥实际作用decode有以下功能支持 UVC、MIPI、RTSP、MP4 四种输入UVC 采用 jpeg_turbo 解码RTSP 使用 gstreamer、mpp 硬解RGA 预处理模型权重复用每路可自定义多线程处理NPU推理自动绑定NPU核心RGA 拼接多路显示墙但性能很烂也完全没有考虑零拷贝6路1080p的RTSP流在RK3588上只有10帧此项目地址rk3588_yolo11_detection设备开发板rk3576/rk3588RTSP模拟器EasyRTSPServer网络摄像头海康威视效果四路1920x1080输入左边两路为UVC输入右边两路为RTSP输入,yolov11sTodork3576下RGA调用可能有越界写目前需要开辟更大的缓冲区防止报错非零拷贝qt5重写显示UI单路单线程优化介绍整体代码分为app、core、modules三部分。app整体控制逻辑core串联模块的核心组件modules主要模块appsrc/app/app_controller.cc是整体控制逻辑利用nlohmann库解析config.json配置文件为每一路配置运行环境启动对应线程src/app/app_controller.ccstd::string build_source_window_name(const std::string window_title, const SourceConfig source_cfg)配置单路运行环境主要是为多线程推理的线程池bool run_single_source(SourceRuntime *runtime)单路运行逻辑bool run_single_source(SourceRuntime *runtime)接受单路配置信息运行此路。SourceBase::Read - DecodeNode - FramePipeline(多线程 infer workerssrc/core/pool/infer_worker.cc) - DisplayNode// 每一路的帧处理流// SourceBase::Read - DecodeNode - FramePipeline(多线程 infer workerssrc/core/pool/infer_worker.cc) - DisplayNodewhile(source_ok!stop_requested()){if(runtime-cfg.fps0.0){constautonowstd::chrono::steady_clock::now();if(nownext_frame_time){// 如果没到的时间就sleep不获取最新帧用于限制最大帧率std::this_thread::sleep_until(next_frame_time);}next_frame_timestd::chrono::steady_clock::now()frame_interval;}// SourceBase::Readcore::types::SourceFrame input_frame;if(!runtime-source-Read(input_frame)){capture_fail_streak;// RTSP 失败概率高constintread_fail_threshold(runtime-cfg.typeINPUT_RTSP)?kRtspReadFailThreshold:1;if(capture_fail_streakread_fail_threshold){LOGW(source read failed (%d/%d before reconnect): %s\n,capture_fail_streak,read_fail_threshold,runtime-cfg.input.c_str());continue;}capture_fail_streak0;LOGE(source frame capture failed: %s\n,runtime-cfg.input.c_str());if(!reconnect_source(runtime,runtime-cfg.input,capture failed)){source_okfalse;keep_error_windowtrue;std::string msgSource reconnect failed\n;msgruntime-cfg.input;show_source_error(runtime,msg);break;}next_frame_timestd::chrono::steady_clock::now()frame_interval;continue;}capture_fail_streak0;// DecodeNodecv::Mat decoded;if(!decode_node.Decode(input_frame,decoded)||decoded.empty()){LOGE(decode failed: %s\n,runtime-cfg.input.c_str());if(!reconnect_source(runtime,runtime-cfg.input,decode failed)){source_okfalse;keep_error_windowtrue;std::string msgSource decode/reconnect failed\n;msgruntime-cfg.input;show_source_error(runtime,msg);break;}continue;}got_frametrue;// 得到至少一个有效帧用于验证此路是否初始化成功constcv::Mat*use_framedecoded;// 更新帧的实际尺寸if(metrics.input_width0metrics.input_height0){metrics.input_widthuse_frame-cols;metrics.input_heightuse_frame-rows;}// 多线程功能生产者-多消费者多生产者-单消费者// 每一路的主线程不断将帧放入生产队列workerpool中worker从队列中取出一帧进行处理随后将处理后的帧放到就绪队列// worker内部逻辑见src/core/pool/infer_worker.cc// 入队前先做容量控制限制队列深度以控制时延和内存。pipeline.WaitForCapacity();pipeline.EnqueueFrame(*use_frame,input_frame.capture_tp);// DisplayNodeFrameResult ready;if(pipeline.PopReady(ready))// 弹出就绪帧{if(handle_ready_frame(runtime,ready,fps_tracker,metrics))// 显示break;}}每一帧在Decode时均在此路的主线程通过 FramePipeline 队列分发给多个infer worker每个worker 执行预处理、推理、后处理随后将帧翻入Result队列core串联模块的核心组件帧队列、线程池和模块间共享数据结构定义还有一些日志工具src/core/log 利用spdlog库实现的日志功能src/core/pipeline多线消费者使用的帧队列src/core/pool线程池和worker内部逻辑自动绑定NPU核心src/core/pool/infer_worker.cc 每一个worker内部逻辑if(task.do_infer){// 标准流程预处理 - NPU 推理 - 后处理框解码与叠加绘制。modules::preprocess::PreprocessOutput preprocess_out;if(preprocess_node.Run(res.frame,worker-ctx,preprocess_out)){core::types::InferOutput infer_out;if(infer_node.Run(worker-ctx,preprocess_out.input_image,infer_out)){modules::postprocess::PostprocessOutput post_out;if(postprocess_node.Run(worker-ctx,infer_out,preprocess_out.letterbox,worker-conf_threshold,res.frame,post_out)){res.infer_msinfer_out.infer_ms;res.detectionspost_out.detection_count;}}}}{// 把处理完成的帧写回结果队列后续由 ResultQueue 做有序合并。std::lock_guardstd::mutexlk(results-mtx);results-items.push_back(std::move(res));}results-cv.notify_one();src/core/queue 队列src/core/types共享数据结构src/core/utils工具modules每个模块source输入源接口src/modules/source/source_base.h阻塞usb_cam_source.ccv4l2支持YUYV和MJPEGmipi_source.cc与USB类似支持YUYV和NV12rtsp_source.ccRTSPh264easy eai示例代码file_source.ccMP4视频文件namespacemodules{namespacesource{usingSourceFramecore::types::SourceFrame;usingSourceFrameFormatcore::types::SourceFrameFormat;classSourceBase{public:virtual~SourceBase()default;virtualboolOpen()0;virtualvoidClose()0;virtualboolRead(SourceFrame*out)0;};}// namespace source}// namespace modulesmGstChn.vDecgst_element_factory_make(mppvideodec,vDec);mGstChn.vScalegst_element_factory_make(videoscale,vScale);mGstChn.vCapsfiltergst_element_factory_make(capsfilter,vCapsfilter);mGstChn.vSinkgst_element_factory_make(appsink,vSink);decode解码但目前更多承担颜色格式转换功能RTSP的解码还是在source下UVC的解码在此处MJPEGboolDecodeMjpeg(constcore::types::SourceFrameinput,cv::Mat*out){if(input.data.empty())returnfalse;if(!tj_handle_){tj_handle_tjInitDecompress();if(!tj_handle_){LOGE(tjInitDecompress failed\n);returnfalse;}}intwidth0;intheight0;intsubsamp0;intcolorspace0;if(tjDecompressHeader3(tj_handle_,input.data.data(),static_castunsignedlong(input.data.size()),width,height,subsamp,colorspace)!0){LOGE(turbojpeg header decode failed: %s\n,tjGetErrorStr2(tj_handle_));returnfalse;}if(width0||height0)returnfalse;out-create(height,width,CV_8UC3);constintpitchwidth*3;if(tjDecompress2(tj_handle_,input.data.data(),static_castunsignedlong(input.data.size()),out-data,width,pitch,height,TJPF_BGR,0)!0){LOGE(turbojpeg decode failed: %s\n,tjGetErrorStr2(tj_handle_));out-release();returnfalse;}returntrue;}preprocess预处理rk示例代码使用RGA的imfill、improcessret_rgaimprocess(rga_buf_src,rga_buf_dst,pat,srect,drect,prect,usage);inference推理其中src/modules/inference/infer_context.cc用于共享权重rk示例代码intinit_infer_context(constchar*model_path,rknn_app_context_t*app_ctx){if(!model_path||!app_ctx)return-1;app_ctx-shared_handlenullptr;app_ctx-rknn_ctx0;app_ctx-input_attrsnullptr;app_ctx-output_attrsnullptr;app_ctx-is_quantfalse;{std::lock_guardstd::mutexlk(g_shared_model_mtx);if(g_shared_model.share_enabledensure_shared_model(model_path,g_shared_model)0){rknn_context ctx0;constintretrknn_dup_context(g_shared_model.base_ctx,ctx);if(retRKNN_SUCC){app_ctx-rknn_ctxctx;fill_app_ctx_from_info(app_ctx,g_shared_model.info);app_ctx-shared_handleg_shared_model;g_shared_model.ref_count;LOGI(reuse model weights: %s (shared contexts%d)\n,model_path,g_shared_model.ref_count);return0;}LOGW(rknn_dup_context failed (ret%d), fallback to per-context init\n,ret);g_shared_model.share_enabledfalse;if(g_shared_model.ref_count0){reset_shared_model(g_shared_model);}}}returninit_context_standalone(model_path,app_ctx);}intrun_forward(rknn_app_context_t*app_ctx,image_buffer_t*preprocessed_img,std::vectorrknn_output*outputs){if(!app_ctx||!preprocessed_img||!outputs){return-1;}rknn_input inputs[app_ctx-io_num.n_input];memset(inputs,0,sizeof(inputs));inputs[0].index0;inputs[0].typeRKNN_TENSOR_UINT8;inputs[0].fmtRKNN_TENSOR_NHWC;inputs[0].sizeapp_ctx-model_width*app_ctx-model_height*app_ctx-model_channel;inputs[0].bufpreprocessed_img-virt_addr;intretrknn_inputs_set(app_ctx-rknn_ctx,app_ctx-io_num.n_input,inputs);if(ret0){LOGE(rknn_input_set fail! ret%d\n,ret);return-1;}retrknn_run(app_ctx-rknn_ctx,nullptr);if(ret0){LOGE(rknn_run fail! ret%d\n,ret);return-1;}outputs-assign(app_ctx-io_num.n_output,rknn_output{});for(inti0;iapp_ctx-io_num.n_output;i){(*outputs)[i].indexi;(*outputs)[i].want_float(!app_ctx-is_quant);}retrknn_outputs_get(app_ctx-rknn_ctx,app_ctx-io_num.n_output,outputs-data(),NULL);if(ret0){LOGE(rknn_outputs_get fail! ret%d\n,ret);outputs-clear();return-1;}return0;}}// namespacepostprocess后处理使用CPUrk示例代码compositor多路显示墙RGA拼接图像easy eai示例代码staticvoidcommitImgtoDispBufMap(intchnId,void*pSrcData,RgaSURF_FORMAT srcFmt,intsrcWidth,intsrcHeight,intsrcHStride,intsrcVStride){if(gChnNums0){return;}intchnNumsgChnNums;intwinWidthgWinWidth;intwinHeightgWinHeight;if(gMutexInited){pthread_mutex_lock(gmutex);chnNumsgChnNums;winWidthgWinWidth;winHeightgWinHeight;pthread_mutex_unlock(gmutex);}if(chnNums0||winWidth0||winHeight0){return;}intunits0;while(1){units;if(chnNums(units*units)){break;}}Image srcImage;Image dstImage;memset(srcImage,0,sizeof(srcImage));memset(dstImage,0,sizeof(dstImage));srcImage.fmtsrcFmt;srcImage.widthsrcWidth;srcImage.heightsrcHeight;srcImage.hor_stridesrcHStride;srcImage.ver_stridesrcVStride;srcImage.rotationHAL_TRANSFORM_ROT_0;srcImage.pBufpSrcData;PTRINT dstBufPtr(PTRINT)*gppDispMapcalcBufMapOffset(chnId,units,winWidth,winHeight);dstImage.fmtRK_FORMAT_RGB_888;dstImage.widthwinWidth/units;dstImage.heightwinHeight/units;dstImage.hor_stridewinWidth;dstImage.ver_stridewinHeight/units;dstImage.rotationHAL_TRANSFORM_ROT_0;dstImage.pBuf(void*)dstBufPtr;srcImg_ConvertTo_dstImg(dstImage,srcImage);}display显示GTKeasy eai示例代码GridCompositorImgDesc_t img_desc{};img_desc.chnIdchannel_id;img_desc.widthsrc.cols;img_desc.heightsrc.rows;img_desc.horStridestatic_castint(src.step/src.elemSize());img_desc.verStridesrc.rows;img_desc.dataSizestatic_castint(src.total()*src.elemSize());strncpy(img_desc.fmt,BGR,sizeof(img_desc.fmt)-1);grid_compositor_submit_frame(reinterpret_castchar*(src.data),img_desc);{GtkWallStatestateWallState();std::lock_guardstd::mutexlk(state.mutex);if(displayIsRunning()){state.display_seen_runningtrue;}if(state.display_seen_running!displayIsRunning()){returntrue;}}