解决部分bug

.vscode/settings.json

@@ -71,7 +71,9 @@
         "hash_set": "cpp",
         "codecvt": "cpp",
         "future": "cpp",
-        "queue": "cpp"
+        "queue": "cpp",
+        "filesystem": "cpp",
+        "__nullptr": "cpp"
     },
     "C_Cpp.errorSquiggles": "Disabled",
     "commentTranslate.targetLanguage": "en"

modules/inference/include/inference.h

@@ -4,7 +4,7 @@
  * @Autor: lishengyin
  * @Date: 2021-10-13 09:37:51
  * @LastEditors: lishengyin
- * @LastEditTime: 2021-12-31 15:45:33
+ * @LastEditTime: 2022-01-04 14:16:41
  */
 #pragma once
 #include <iostream>
@@ -80,6 +80,7 @@ namespace MIVA{
         std::shared_ptr<std::vector<DataSource>> m_DataList = NULL;
         std::shared_ptr<InferInfo> m_InferInfo = nullptr;
         std::shared_ptr<recorder> m_recorder = nullptr;
+        
         bool Play = false;
         bool enable = false;
 
@@ -127,14 +128,6 @@ namespace MIVA{
         int32_t StartTask();
 
         /**
-         * @description: 暂停任务
-         * @param {*}
-         * @return {*}
-         * @author: lishengyin
-         */        
-        void PauseTask();
-
-        /**
          * @description: 停止任务
          * @param {*}
          * @return {*}

modules/inference/src/inference.cpp

@@ -4,7 +4,7 @@
  * @Autor: lishengyin
  * @Date: 2021-10-13 09:35:37
  * @LastEditors: lishengyin
- * @LastEditTime: 2021-12-31 17:37:04
+ * @LastEditTime: 2022-01-04 16:12:54
  */
 #include "inference.h"
 
@@ -53,7 +53,9 @@ GMutex eos_lock;
 GstElement *g_streammux = NULL;
 std::shared_ptr<InferInfo> g_InferInfo = NULL;
 
-map<string, map<int, queue<cv::Mat>>> m_frames;
+map<string, map<int, queue<std::shared_ptr<cv::Mat>>>> m_frames;
+
+bool RecordEnable = false;
 
 namespace MIVA{
     std::shared_ptr<Inference> infer = NULL;
@@ -75,7 +77,10 @@ namespace MIVA{
     }
     Inference::~Inference()
     {
-        DebugL << "Returned, stopping playback";
+        DebugL << "Returned, stopping playback" << endl;
+        GstState state;
+        gst_element_get_state(this->pipeline,&state,NULL, GST_CLOCK_TIME_NONE);
+        if(state == GST_STATE_PLAYING)  gst_element_set_state(this->pipeline, GST_STATE_PAUSED);
         this->m_InferInfo = NULL;
         gst_element_set_state(this->pipeline, GST_STATE_NULL);
         g_main_loop_quit (this->loop);
@@ -134,35 +139,35 @@ namespace MIVA{
         this->queue2 = gst_element_factory_make ("queue", "queue2");
         this->queue3 = gst_element_factory_make ("queue", "queue3");
         this->queue4 = gst_element_factory_make ("queue", "queue4");
-        this->queue5 = gst_element_factory_make ("queue", "queue5");
+        // this->queue5 = gst_element_factory_make ("queue", "queue5");
 
 
         /* Use nvtiler to composite the batched frames into a 2D tiled array based
          * on the source of the frames. */
-        this->tiler = gst_element_factory_make ("nvmultistreamtiler", "nvtiler");
+        //this->tiler = gst_element_factory_make ("nvmultistreamtiler", "nvtiler");
 
         /* Use convertor to convert from NV12 to RGBA as required by nvosd */
         this->nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter");
 
         this->nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay");
 
-        #ifdef PLATFORM_TEGRA
-            this->transform = gst_element_factory_make ("nvegltransform", "nvegl-transform");
-        #endif
+        // #ifdef PLATFORM_TEGRA
+        //     this->transform = gst_element_factory_make ("nvegltransform", "nvegl-transform");
+        // #endif
 
         this->sink = gst_element_factory_make ("fakesink", "nvvideo-renderer");
         
-        if (!this->pgie || !this->tiler || !this->nvvidconv || !this->nvosd || !this->sink) {
+        if (!this->pgie || !this->nvvidconv || !this->nvosd || !this->sink) {
             ErrorL << "One element could not be created. Exiting.";
             return -1;
         }
 
-        #ifdef PLATFORM_TEGRA
-            if(!this->transform) {
-                ErrorL << "One tegra element could not be created. Exiting.";
-                return -1;
-            }
-        #endif
+        // #ifdef PLATFORM_TEGRA
+        //     if(!this->transform) {
+        //         ErrorL << "One tegra element could not be created. Exiting.";
+        //         return -1;
+        //     }
+        // #endif
    
         g_object_set(G_OBJECT(streammux), "batch-size", g_num_sources, NULL);
 
@@ -183,12 +188,12 @@ namespace MIVA{
             g_object_set (G_OBJECT (this->pgie), "batch-size", g_num_sources, NULL);
         }
 
-        this->tiler_rows = (guint) sqrt (g_num_sources);
-        this->tiler_columns = (guint) ceil (1.0 * g_num_sources / this->tiler_rows);
+        // this->tiler_rows = (guint) sqrt (g_num_sources);
+        // this->tiler_columns = (guint) ceil (1.0 * g_num_sources / this->tiler_rows);
 
-        /* we set the tiler properties here */
-        g_object_set (G_OBJECT (this->tiler), "rows", this->tiler_rows, "columns", this->tiler_columns,
-            "width", TILED_OUTPUT_WIDTH, "height", TILED_OUTPUT_HEIGHT, NULL);
+        // /* we set the tiler properties here */
+        // g_object_set (G_OBJECT (this->tiler), "rows", this->tiler_rows, "columns", this->tiler_columns,
+        //     "width", TILED_OUTPUT_WIDTH, "height", TILED_OUTPUT_HEIGHT, NULL);
 
 
         g_object_set (G_OBJECT (this->nvosd), "process-mode", OSD_PROCESS_MODE,
@@ -202,13 +207,14 @@ namespace MIVA{
         gst_object_unref (this->bus);
         
         gst_bin_add_many (GST_BIN (this->pipeline), this->queue1, this->pgie, this->queue2, this->nvvidconv, this->queue3,
-                this->tiler, this->queue4,this->nvosd, this->queue5, this->sink, NULL);
+                this->nvosd, this->queue4, this->sink, NULL);
 
         if (!gst_element_link_many (streammux, this->queue1, this->pgie, this->queue2, this->nvvidconv, this->queue3,
-                    this->tiler, this->queue4,this->nvosd, this->queue5, this->sink, NULL)) {
+                    this->nvosd, this->queue4, this->sink, NULL)) {
             ErrorL << "Elements could not be linked. Exiting.";
             return -1;
         }
+        // this->tiler, this->queue4,
     
         this->tiler_sink_pad = gst_element_get_static_pad (this->queue3, "sink");
         gst_pad_add_probe (this->tiler_sink_pad, GST_PAD_PROBE_TYPE_BUFFER,
@@ -246,14 +252,13 @@ namespace MIVA{
         DebugL << "state:" << state << endl;
         if(state == GST_STATE_NULL){
             pool.async([&](){
-                this->RestartTask();
-                if(m_recorder != nullptr) m_recorder->Start();
+                if(m_recorder != nullptr && this->RestartTask() == OK) m_recorder->Start();
+                // 不能去掉, 去掉会触发Deepstream内部句柄无法释放的Bug
                 g_main_loop_run(this->loop);
             });
         }else{
             pool.async([&](){
-                this->RestartTask();
-                if(m_recorder != nullptr) m_recorder->Start();
+                if(m_recorder != nullptr && this->RestartTask() == OK) m_recorder->Start();
             });
         }
         pool.start();
@@ -271,23 +276,6 @@ namespace MIVA{
     }
 
     /**
-     * @description: 暂停任务
-     * @param {*}
-     * @return {*}
-     * @author: lishengyin
-     */  
-    void Inference::PauseTask()
-    {
-        InfoL << "Now Pause";
-        std::vector<DataSource>::iterator iter;
-        for(iter = this->m_InferInfo->DataSources.begin(); iter != this->m_InferInfo->DataSources.end(); iter++){
-            if(iter->Play && iter->source_bin != NULL){
-                gst_element_set_state(iter->source_bin, GST_STATE_PAUSED);
-            }
-        }
-    }
-
-    /**
      * @description: 停止任务
      * @param {*}
      * @return {*}
@@ -298,6 +286,7 @@ namespace MIVA{
         int sourceId = 0;
         GstStateChangeReturn state_return;
         std::vector<DataSource>::iterator iter;
+        // gst_element_set_state(this->pipeline, GST_STATE_PAUSED);
         for (iter = this->m_InferInfo->DataSources.begin(); iter != this->m_InferInfo->DataSources.end(); iter++){
             if(iter->source_bin != NULL){
                 state_return = gst_element_set_state(iter->source_bin, GST_STATE_NULL);
@@ -320,10 +309,6 @@ namespace MIVA{
             }
             sourceId++;
         }
-        //gst_element_set_state(this->streammux, GST_STATE_NULL);
-        // gst_element_set_state(this->nvvidconv, GST_STATE_NULL);
-        // gst_element_set_state(this->nvosd, GST_STATE_NULL);
-        // gst_element_set_state(this->tiler, GST_STATE_NULL);
 
         if(m_recorder != nullptr){
             m_recorder->Finish();
@@ -352,8 +337,6 @@ namespace MIVA{
             return ERR;
         }
         DebugL << "RestartTask" << endl;
-        
-   
         GstState state;
         gst_element_get_state(this->pipeline,&state,NULL, GST_CLOCK_TIME_NONE);
         DebugL << "state:" << state << endl;
@@ -412,116 +395,136 @@ namespace MIVA{
         NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
         l_frame = batch_meta->frame_meta_list; 
 
-        map<string, vector<cv::Mat>> frames;
-        frames.clear();
-        int i = 0;
         for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;l_frame = l_frame->next) //从批量中获取某一帧图
         {
             NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
             int batch_id = frame_meta->batch_id;
             /* Validate user meta */
-            src_data = (char*) malloc(surface->surfaceList[frame_meta->batch_id].dataSize);
-            if(src_data == NULL) {
-                g_print("Error: failed to malloc src_data \n");
-            }
+    
             NvBufSurfacePlaneParams *pParams = &surface->surfaceList[batch_id].planeParams;
-            #ifdef PLATFORM_TEGRA
-                NvBufSurfaceMap(surface, -1, -1, NVBUF_MAP_READ);
-                unsigned int offset = 0;
-                for (unsigned int num_planes = 0; num_planes < pParams->num_planes; num_planes++)
+
+            // 识别区域
+            std::string json = g_InferInfo->DataSources[frame_meta->source_id].range;
+            InferRange inferRange;
+            int num = 0;
+
+            if(inferRange.jsonToObject(json)){
+                for (l_obj = frame_meta->obj_meta_list; l_obj != NULL;l_obj = l_obj->next) 
                 {
-                    if (num_planes > 0)
-                        offset += pParams->height[num_planes - 1] * (pParams->bytesPerPix[num_planes - 1] * pParams->width[num_planes - 1]);
-                    for (unsigned int h = 0; h < pParams->height[num_planes]; h++)
+                    obj_meta = (NvDsObjectMeta *) (l_obj->data);
+                    if (obj_meta->class_id == 0) {
+                        NvDsComp_BboxInfo boxInfo;
+                        boxInfo = obj_meta->detector_bbox_info;
+                        NvBbox_Coords box_Coord;
+                        box_Coord = boxInfo.org_bbox_coords;
+                        Point p;
+                        p.x = box_Coord.left + box_Coord.width / 2;
+                        p.y = box_Coord.top + box_Coord.height / 2;
+                        if(Inference::PtInPolygon(p, inferRange.m_points, inferRange.m_points.size())) num++;
+                    }
+                }
+            }else{
+                for (l_obj = frame_meta->obj_meta_list; l_obj != NULL;l_obj = l_obj->next) 
+                {
+                    obj_meta = (NvDsObjectMeta *) (l_obj->data);
+                    if (obj_meta->class_id == 0) {
+                         num++;
+                    }
+                }
+            }
+            
+            if(RecordEnable){
+                if(src_data == NULL) {
+                    src_data = (char*) malloc(surface->surfaceList[frame_meta->batch_id].dataSize);
+                }
+            }
+            #ifdef PLATFORM_TEGRA
+                if(src_data != NULL && RecordEnable){
+                    NvBufSurfaceMap(surface, -1, -1, NVBUF_MAP_READ);
+                    unsigned int offset = 0;
+                    for (unsigned int num_planes = 0; num_planes < pParams->num_planes; num_planes++)
                     {
-                        memcpy((void *)(src_data + offset + h * pParams->bytesPerPix[num_planes] * pParams->width[num_planes]),
-                            (void *)((char *)surface->surfaceList[batch_id].mappedAddr.addr[num_planes] + h * pParams->pitch[num_planes]),
-                            pParams->bytesPerPix[num_planes] * pParams->width[num_planes]);
+                        if (num_planes > 0)
+                            offset += pParams->height[num_planes - 1] * (pParams->bytesPerPix[num_planes - 1] * pParams->width[num_planes - 1]);
+                        for (unsigned int h = 0; h < pParams->height[num_planes]; h++)
+                        {
+                            memcpy((void *)(src_data + offset + h * pParams->bytesPerPix[num_planes] * pParams->width[num_planes]),
+                                (void *)((char *)surface->surfaceList[batch_id].mappedAddr.addr[num_planes] + h * pParams->pitch[num_planes]),
+                                pParams->bytesPerPix[num_planes] * pParams->width[num_planes]);
+                        }
                     }
+                    NvBufSurfaceSyncForDevice(surface, -1, -1);
+                    NvBufSurfaceUnMap(surface, -1, -1);
                 }
-                NvBufSurfaceSyncForDevice(surface, -1, -1);
-                NvBufSurfaceUnMap(surface, -1, -1);
             #else
                 cudaMemcpy((void*)src_data,
                             (void*)surface->surfaceList[frame_meta->batch_id].dataPtr,
                             surface->surfaceList[frame_meta->batch_id].dataSize,
                             cudaMemcpyDeviceToHost);
             #endif 
+
             gint frame_width = (gint)surface->surfaceList[frame_meta->batch_id].width;
             gint frame_height = (gint)surface->surfaceList[frame_meta->batch_id].height;
             gint frame_step = surface->surfaceList[frame_meta->batch_id].pitch;
-            
-            // 类型转换
-            cv::Mat frame = cv::Mat(frame_height, frame_width, CV_8UC4, src_data, frame_step);
-            cv::Mat out_mat = cv::Mat (cv::Size(frame_width, frame_height), CV_8UC3);
-            cv::cvtColor(frame, out_mat, CV_RGBA2BGR);
 
-            int num = 0;
-            // OSD
-            for (l_obj = frame_meta->obj_meta_list; l_obj != NULL;l_obj = l_obj->next) 
-            {
-                obj_meta = (NvDsObjectMeta *) (l_obj->data);
-                if (obj_meta->class_id == 0) {
-                    NvDsComp_BboxInfo boxInfo;
-                    boxInfo = obj_meta->detector_bbox_info;
-                    NvBbox_Coords box_Coord;
-                    box_Coord = boxInfo.org_bbox_coords;
-                    cv::Point top_left;
-                    cv::Point bottom_right;
-                    top_left.x = box_Coord.left;
-                    top_left.y = box_Coord.top;
-                    bottom_right.x = box_Coord.left + box_Coord.width;
-                    bottom_right.y = box_Coord.top +  box_Coord.height;
-                    cv::Scalar color(100, 200, 0);
-                    cv::rectangle(out_mat, top_left, bottom_right, color, 3);
-                }
+            int flag = 1;
+            if(!RecordEnable || src_data == NULL) flag = 0;
+            if(RecordEnable){
+                if(m_frames[g_InferInfo->DataSources[frame_meta->source_id].Pid][frame_meta->source_id].size() != 0) flag = 0;
             }
-            // 识别区域
-            std::string json = g_InferInfo->DataSources[frame_meta->source_id].range;
-            InferRange inferRange;
-            if(inferRange.jsonToObject(json)){
-                if(inferRange.m_points.size() > 0){
-                    for(int i = 0; i < (int)(inferRange.m_points.size()); i++)
-                    {
-                        cv::Point p1;
-                        cv::Point p2;
-                        p1.x = inferRange.m_points[i].x;
-                        p1.y = inferRange.m_points[i].y;
-
-                        if((i + 1) < (int)(inferRange.m_points.size())){
-                            p2.x = inferRange.m_points[i+1].x;
-                            p2.y = inferRange.m_points[i+1].y;
-                        }else{
-                            p2.x = inferRange.m_points[0].x;
-                            p2.y = inferRange.m_points[0].y;
-                        }
+            
+            if(flag){
+                // 提取数据
+                std::shared_ptr<cv::Mat> frame = std::make_shared<cv::Mat>(frame_height, frame_width, CV_8UC4, src_data, frame_step);
+                // 类型转换
+                cv::cvtColor(*frame, *frame, CV_RGBA2BGR);
+                // OSD
+                for (l_obj = frame_meta->obj_meta_list; l_obj != NULL;l_obj = l_obj->next) 
+                {
+                    obj_meta = (NvDsObjectMeta *) (l_obj->data);
+                    if (obj_meta->class_id == 0) {
+                        NvDsComp_BboxInfo boxInfo;
+                        boxInfo = obj_meta->detector_bbox_info;
+                        NvBbox_Coords box_Coord;
+                        box_Coord = boxInfo.org_bbox_coords;
+                        cv::Point top_left;
+                        cv::Point bottom_right;
+                        top_left.x = box_Coord.left;
+                        top_left.y = box_Coord.top;
+                        bottom_right.x = box_Coord.left + box_Coord.width;
+                        bottom_right.y = box_Coord.top +  box_Coord.height;
                         cv::Scalar color(100, 200, 0);
-                        cv::line(out_mat, p1, p2, color, 4);
+                        cv::rectangle(*frame, top_left, bottom_right, color, 3);
                     }
-
-                    for (l_obj = frame_meta->obj_meta_list; l_obj != NULL;l_obj = l_obj->next) 
-                    {
-                        obj_meta = (NvDsObjectMeta *) (l_obj->data);
-                        if (obj_meta->class_id == 0) {
-                            NvDsComp_BboxInfo boxInfo;
-                            boxInfo = obj_meta->detector_bbox_info;
-                            NvBbox_Coords box_Coord;
-                            box_Coord = boxInfo.org_bbox_coords;
-                            Point p;
-                            p.x = box_Coord.left + box_Coord.width / 2;
-                            p.y = box_Coord.top + box_Coord.height / 2;
-                            if(Inference::PtInPolygon(p, inferRange.m_points, inferRange.m_points.size())) num++;
+                }
+                if(inferRange.jsonToObject(json)){
+                    if(inferRange.m_points.size() > 0){
+                        for(int i = 0; i < (int)(inferRange.m_points.size()); i++)
+                        {
+                            cv::Point p1;
+                            cv::Point p2;
+                            p1.x = inferRange.m_points[i].x;
+                            p1.y = inferRange.m_points[i].y;
+
+                            if((i + 1) < (int)(inferRange.m_points.size())){
+                                p2.x = inferRange.m_points[i+1].x;
+                                p2.y = inferRange.m_points[i+1].y;
+                            }else{
+                                p2.x = inferRange.m_points[0].x;
+                                p2.y = inferRange.m_points[0].y;
+                            }
+                            cv::Scalar color(100, 200, 0);
+                            cv::line(*frame, p1, p2, color, 4);
                         }
                     }
                 }
+                m_frames[g_InferInfo->DataSources[frame_meta->source_id].Pid][frame_meta->source_id].push(frame);
             }
-            m_frames[g_InferInfo->DataSources[frame_meta->source_id].Pid][frame_meta->source_id].push(out_mat);
             NoticeCenter::Instance().emitEvent(NOTICE_INFER,frame_meta->source_id, num);
-            i++;
-            if(src_data != NULL) {
-                free(src_data);
-                src_data = NULL;
-            }
+        }
+        if(src_data != NULL) {
+            free(src_data);
+            src_data = NULL;
         }
         gst_buffer_unmap (buf, &in_map_info);
         return GST_PAD_PROBE_OK;
@@ -538,9 +541,12 @@ namespace MIVA{
                         num = source.AddrNum;
                     }
                 }
-                if(num != (int)iter->second.size()) continue;
+                if(num != (int)iter->second.size()){
+                    continue;
+                } 
                 cv::Mat outImg;
-                cv::Mat img;
+
+                std::shared_ptr<cv::Mat> img;
                 int rows = (int) sqrt (num);
                 int columns = (int) ceil (1.0 * num / rows);
                 
@@ -553,7 +559,7 @@ namespace MIVA{
                             img = source->second.front();
                             source->second.pop();
                         }else{
-                            hconcat(img,source->second.front(),img);
+                            hconcat(*img,*(source->second.front()),*img);
                             source->second.pop();
                         }
                         if(source != iter->second.end()){
@@ -561,8 +567,8 @@ namespace MIVA{
                             count++;
                         }
                     }
-                    if(row == 0) outImg = img;
-                    else vconcat(outImg,img,outImg);
+                    if(row == 0) outImg = *img;
+                    else vconcat(outImg,*img,outImg);
                 }
                 cv::resize(outImg, outImg, cv::Size(1920, 1080));
                 m_recorder->ConsumerData(iter->first, outImg);
@@ -661,7 +667,7 @@ namespace MIVA{
                         InfoL << "Got EOS from stream " << stream_id;
                     }
                 }
-            break;
+                break;
             }
         #endif
             default:

modules/userApp/src/user_app.cpp

@@ -4,7 +4,7 @@
  * @Autor: lishengyin
  * @Date: 2021-10-13 09:35:42
  * @LastEditors: lishengyin
- * @LastEditTime: 2021-12-31 15:59:14
+ * @LastEditTime: 2022-01-04 16:09:50
  */
 #include "user_app.h"
 
@@ -285,10 +285,10 @@ namespace MIVA
     void UserApp::ListenInferPer()
     {
         if(this->m_Infer->Play == true){
-            this->m_Infer->StopTask();
-            // 推理结束发布InferData事件
+             // 推理结束发布InferData事件
             NoticeCenter::Instance().emitEvent(NOTICE_INFERDATA);
             this->m_Infer->Recorder();
+            this->m_Infer->StopTask();
         }
     }