初版全部功能

CMakeLists.txt

@@ -131,7 +131,7 @@ link_directories(/usr/local/lib/)
 
 
 list(APPEND Source_LINKER_LIBS ZLToolKit dl ${3RDPARTY_LIBS}  ${OpenCV_LIBS} ${FFMPEG_LIBRARIES} pthread)
-list(APPEND ALL_LINKER_LIBS gstreamer-1.0 gobject-2.0 glib-2.0 nvdsgst_meta nvds_meta nvdsgst_helper nvbufsurface m ${Source_LINKER_LIBS}) 
+list(APPEND ALL_LINKER_LIBS gstrtspserver-1.0 gstreamer-1.0 gobject-2.0 glib-2.0 nvdsgst_meta nvds_meta nvdsgst_helper nvbufsurface m ${Source_LINKER_LIBS}) 
 
 add_subdirectory(${PROJECT_SOURCE_DIR}/modules)
 

modules/inference/include/inference.h

@@ -4,7 +4,7 @@
  * @Autor: lishengyin
  * @Date: 2021-10-13 09:37:51
  * @LastEditors: lishengyin
- * @LastEditTime: 2022-07-12 09:46:50
+ * @LastEditTime: 2022-07-13 16:54:01
  */
 #pragma once
 #include <iostream>
@@ -64,8 +64,12 @@ namespace gsd_ds{
         // Deepstream
         GMainLoop *loop = NULL;
         GstElement *pipeline = NULL,*streammux =NULL, *sink = NULL, *pgie = NULL,
-        *queue1, *queue2, *queue3, *queue4, *queue5, *nvvidconv = NULL,
+        *queue1, *queue2, *queue3, *queue4, *queue5, *nvvidconv = NULL, *caps_filter = NULL, *nvvidconv_postosd = NULL,
+        *encoder = NULL, 
+        *codecparser = NULL, 
+        *rtppay = NULL, 
         *nvosd = NULL,*tiler = NULL;
+
         GstElement *transform = NULL;
         GstBus *bus = NULL;
         guint bus_watch_id;
@@ -155,6 +159,12 @@ namespace gsd_ds{
          */        
         bool getAlive();
 
+        /**
+         * @description: 动态调节
+         * @return {*}
+         */        
+        void DynamicAdjust();
+
     public:
 
         /**

modules/inference/src/inference.cpp

@@ -4,11 +4,15 @@
  * @Autor: lishengyin
  * @Date: 2021-10-13 09:35:37
  * @LastEditors: lishengyin
- * @LastEditTime: 2022-07-13 11:11:37
+ * @LastEditTime: 2022-07-14 09:59:36
  */
 #include "inference.h"
 #include "Notices.h"
 
+#include <gst/rtsp-server/rtsp-server.h>
+#include <gst/rtsp/gstrtsptransport.h>
+#include <gst/rtp/gstrtcpbuffer.h>
+
 #define MAX_DISPLAY_LEN 64
 
 #define PGIE_CLASS_ID_VEHICLE 0
@@ -58,6 +62,57 @@ map<string, map<int, queue<std::shared_ptr<cv::Mat>>>> m_frames;
 
 std::vector<cv::Scalar> colors_;
 
+#define MAX_SINK_BINS (1024)
+
+static guint server_count = 0;
+
+static GstRTSPServer *server [MAX_SINK_BINS];
+static GMutex server_cnt_lock;
+
+static gboolean
+start_rtsp_streaming (guint rtsp_port_num, guint updsink_port_num, guint64 udp_buffer_size)
+{
+  GstRTSPMountPoints *mounts;
+  GstRTSPMediaFactory *factory;
+  char udpsrc_pipeline[512];
+  char port_num_Str[64] = { 0 };
+
+  if (udp_buffer_size == 0)
+    udp_buffer_size = 512 * 1024;
+
+  sprintf (udpsrc_pipeline,
+      "( udpsrc name=pay0 port=%d buffer-size=%lu caps=\"application/x-rtp, media=video, "
+      "clock-rate=90000, encoding-name=%s, payload=96 \" )",
+      updsink_port_num, udp_buffer_size, "H264");
+
+  sprintf (port_num_Str, "%d", rtsp_port_num);
+
+  g_mutex_lock (&server_cnt_lock);
+
+  server [server_count] = gst_rtsp_server_new ();
+  g_object_set (server [server_count], "service", port_num_Str, NULL);
+
+  mounts = gst_rtsp_server_get_mount_points (server [server_count]);
+
+  factory = gst_rtsp_media_factory_new ();
+  gst_rtsp_media_factory_set_launch (factory, udpsrc_pipeline);
+
+  gst_rtsp_mount_points_add_factory (mounts, "/ds-test", factory);
+
+  g_object_unref (mounts);
+
+  gst_rtsp_server_attach (server [server_count], NULL);
+
+  server_count++;
+
+  g_mutex_unlock (&server_cnt_lock);
+
+  g_print
+      ("\n *** DeepStream: Launched RTSP Streaming at rtsp://localhost:%d/ds-test ***\n\n",
+      rtsp_port_num);
+
+  return TRUE;
+}
 
 // Keep 2 digits after decimal
 static std::string FloatToString(float number) {
@@ -137,6 +192,8 @@ namespace gsd_ds{
             return ERR;
         }
 
+        GstCaps *caps = NULL;
+
         gst_bin_add (GST_BIN (this->pipeline), streammux);
         this->m_InferInfo = InferInfo::CreateNew();
         g_InferInfo = this->m_InferInfo;
@@ -166,20 +223,57 @@ namespace gsd_ds{
 
         /* Use nvtiler to composite the batched frames into a 2D tiled array based
          * on the source of the frames. */
-        //this->tiler = gst_element_factory_make ("nvmultistreamtiler", "nvtiler");
+        this->tiler = gst_element_factory_make ("nvmultistreamtiler", "nvtiler");
 
         /* Use convertor to convert from NV12 to RGBA as required by nvosd */
         this->nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter");
 
+        this->nvvidconv_postosd = gst_element_factory_make("nvvideoconvert", "nvvidconv_postosd");
+
         this->nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay");
 
-        this->sink = gst_element_factory_make ("fakesink", "nvvideo-renderer");
-        
+        g_object_set (G_OBJECT (nvosd), "display-text", 1, NULL);
+        g_object_set (G_OBJECT (nvosd), "display-clock", 1, NULL);
+        g_object_set (G_OBJECT (nvosd), "display-bbox", 1, NULL);
+        g_object_set (G_OBJECT (nvosd), "display-mask", 1, NULL);
+        g_object_set (G_OBJECT (nvosd), "process-mode", 2, NULL);
+
+        this->caps_filter = gst_element_factory_make ("capsfilter", "capsfilter");
+        caps = gst_caps_from_string ("video/x-raw(memory:NVMM), format=I420");
+        this->encoder = gst_element_factory_make ("nvv4l2h264enc", "encoder");
+
+        guint profile = 0;
+        guint bitrate = 4000000;
+        guint iframeinterval = 40;
+
+        this->codecparser = gst_element_factory_make ("h264parse", "h264-parser2");
+        this->rtppay = gst_element_factory_make ("rtph264pay", "rtppay");
+        this->sink = gst_element_factory_make ("udpsink", "udpsink");
+
+        g_object_set (G_OBJECT (caps_filter), "caps", caps, NULL);
+        g_object_set (G_OBJECT (encoder), "profile", profile, NULL);
+        g_object_set (G_OBJECT (encoder), "iframeinterval", iframeinterval, NULL);
+        g_object_set (G_OBJECT (encoder), "bitrate", bitrate, NULL);
+        g_object_set (G_OBJECT (encoder), "preset-level", 1, NULL);
+        g_object_set (G_OBJECT (encoder), "insert-sps-pps", 1, NULL);
+        g_object_set (G_OBJECT (encoder), "bufapi-version", 1, NULL);
+        g_object_set (G_OBJECT (sink), "host", "224.224.255.255", "port",
+            5400, "async", FALSE, "sync", 0, NULL);
+
         if (!this->pgie || !this->nvvidconv || !this->nvosd || !this->sink) {
             ErrorL << "One element could not be created. Exiting.";
             return -1;
         }
 
+        if (!caps_filter || !encoder || !codecparser ) {
+            g_printerr ("One element could not be created. Exiting.\n");
+            return -1;
+        }
+        if (!rtppay) {
+            g_printerr ("rtppay could not be created. Exiting.\n");
+            return -1;
+        }
+
         g_object_set(G_OBJECT(streammux), "batch-size", g_num_sources, NULL);
 
         g_object_set(G_OBJECT(streammux), "live-source", 1, NULL);
@@ -199,39 +293,29 @@ namespace gsd_ds{
             g_object_set (G_OBJECT (this->pgie), "batch-size", g_num_sources, NULL);
         }
         
-        g_object_set (G_OBJECT (this->nvosd), "process-mode", OSD_PROCESS_MODE,
-            "display-text", OSD_DISPLAY_TEXT, NULL);
-        g_object_set (G_OBJECT (this->nvosd), "display-bbox", 0, NULL);
-     
         g_object_set (G_OBJECT (this->sink), "qos", 0, NULL);
+        g_object_set (G_OBJECT (this->sink), "async", false, NULL);
+        g_object_set (G_OBJECT (this->sink), "sync", 1, NULL);
 
         this->bus = gst_pipeline_get_bus (GST_PIPELINE (this->pipeline));
         this->bus_watch_id = gst_bus_add_watch (this->bus, bus_call, this->loop);
         gst_object_unref (this->bus);
         
+        gst_bin_add_many (GST_BIN (this->pipeline), this->queue1, this->pgie, this->tiler, this->queue2, this->nvvidconv, this->nvosd, this->nvvidconv_postosd,this->queue3,this->caps_filter,this->queue4,this->encoder, this->rtppay, this->sink, NULL);
 
-        gst_bin_add_many (GST_BIN (this->pipeline), this->queue1, this->pgie, this->queue2, this->nvvidconv, this->queue3,
-                this->nvosd, this->queue4,this->sink, NULL);
-
-        if (!gst_element_link_many (streammux, this->queue1, this->pgie, this->queue2, this->nvvidconv, this->queue3,
-                    this->nvosd, this->queue4,this->sink, NULL)) {
+        if (!gst_element_link_many (streammux, this->queue1, this->pgie, this->tiler, this->queue2,this->nvvidconv, this->nvosd, this->nvvidconv_postosd,this->queue3,this->caps_filter,this->queue4, this->encoder, this->rtppay, this->sink, NULL)) {
             ErrorL << "Elements could not be linked. Exiting.";
             return -1;
         }
 
-        // gst_bin_add_many (GST_BIN (this->pipeline), this->queue1, this->nvvidconv, this->queue3,
-        //         this->nvosd, this->queue4,this->sink, NULL);
-
-        // if (!gst_element_link_many (streammux, this->queue1, this->nvvidconv, this->queue3,
-        //             this->nvosd, this->queue4,this->sink, NULL)) {
-        //     ErrorL << "Elements could not be linked. Exiting.";
-        //     return -1;
-        // }
+        gboolean ret = TRUE;
+        ret = start_rtsp_streaming (8554, 5400, 0);
 
+        this->DynamicAdjust();
+        
         this->tiler_sink_pad = gst_element_get_static_pad (this->nvvidconv, "src");
         gst_pad_add_probe (this->tiler_sink_pad, GST_PAD_PROBE_TYPE_BUFFER,
                 tiler_sink_pad_buffer_probe, NULL, NULL);
-
         gst_object_unref (this->tiler_sink_pad);
 
         this->enable = true;
@@ -305,16 +389,27 @@ namespace gsd_ds{
     int32_t Inference::RestartTask()
     {
         int num = 0;
+        GstStateChangeReturn state_return;
         std::vector<DataSource>::iterator iter;
         for(iter=this->m_InferInfo->DataSources.begin(); iter!=this->m_InferInfo->DataSources.end();iter++){
             if(iter->Play && iter->source_bin != NULL) num++;
         }
+        for (iter = this->m_InferInfo->DataSources.begin(); iter != this->m_InferInfo->DataSources.end(); iter++){
+            if(iter->source_bin != NULL){
+                state_return = gst_element_set_state(iter->source_bin, GST_STATE_NULL);
+                StateResulit(state_return);
+            }
+        }
+        state_return = gst_element_set_state(this->streammux, GST_STATE_NULL);
+        StateResulit(state_return);
+
         if(num == 0){
             DebugL << "没有数据源需要播放" << endl;
             this->Play = false;
             return ERR;
         }
         DebugL << "StartTask" << endl;
+
         gst_element_set_state(this->pipeline, GST_STATE_PAUSED);
         gst_element_set_state(this->pipeline, GST_STATE_PLAYING);
         m_frames.clear();
@@ -404,8 +499,9 @@ namespace gsd_ds{
                 if (obj_meta->class_id == 14) {
                     num++;
                 }
-            }
+            } 
             NoticeCenter::Instance().emitEvent(NOTICE_DEV, frame_meta->source_id, num);
+
             if(src_data == nullptr) {
                 src_data = (char*) malloc(surface->surfaceList[frame_meta->batch_id].dataSize);
             }
@@ -433,6 +529,7 @@ namespace gsd_ds{
                             surface->surfaceList[frame_meta->batch_id].dataSize,
                             cudaMemcpyDeviceToHost);
             #endif 
+            
             gint frame_width = (gint)surface->surfaceList[frame_meta->batch_id].width;
             gint frame_height = (gint)surface->surfaceList[frame_meta->batch_id].height;
             gint frame_step = surface->surfaceList[frame_meta->batch_id].pitch;
@@ -466,8 +563,6 @@ namespace gsd_ds{
             if(cnStreamInferData->Objects.empty()) continue;
             std::string json;
             cnStreamInferData->objectToJson(json);
-            DebugL << json << endl;
-
             std::vector<uint8_t> buf;
             cv::imencode(".jpg",*frame,buf);
             auto *enc_msg = reinterpret_cast<unsigned char*>(buf.data());
@@ -745,12 +840,12 @@ namespace gsd_ds{
     void Inference::stop_release_source (gint source_id)
     {
         g_mutex_lock (&eos_lock);
-
+        gchar pad_name[16];
+        GstPad *sinkpad = NULL;
         GstState state;
         gst_element_get_state(this->m_InferInfo->DataSources[source_id].source_bin,&state,NULL, GST_CLOCK_TIME_NONE);
         // DebugL << "state:" << state << endl;
         WarnL << "stop_source ID:( " << source_id << " , " << this->m_InferInfo->DataSources[source_id].uri << ")" << endl;
-
         if(state == GST_STATE_NULL){
             gst_bin_remove (GST_BIN (this->pipeline), this->m_InferInfo->DataSources[source_id].source_bin);
         }else{
@@ -759,6 +854,10 @@ namespace gsd_ds{
             switch (state_return) {
                 case GST_STATE_CHANGE_SUCCESS:
                     InfoL << "STATE CHANGE SUCCESS";
+                    // g_snprintf (pad_name, 15, "sink_%u", source_id);
+                    // sinkpad = gst_element_get_static_pad (this->streammux, pad_name);
+                    // gst_pad_send_event (sinkpad, gst_event_new_flush_stop (FALSE));
+                    // gst_element_release_request_pad (this0>streammux, sinkpad);
                     gst_bin_remove (GST_BIN (this->pipeline), this->m_InferInfo->DataSources[source_id].source_bin);
                     DebugL << "该source_bin移除成功" << endl;
                     break;
@@ -906,4 +1005,19 @@ namespace gsd_ds{
             }
         }
     }
+    
+    /**
+     * @description: 动态调节
+     * @return {*}
+     */    
+    void Inference::DynamicAdjust(){
+        int num = 0;
+        // std::vector<DataSource>::iterator iter;
+        // for(iter=this->m_InferInfo->DataSources.begin(); iter!=this->m_InferInfo->DataSources.end();iter++){
+        //     if(iter->Play && iter->source_bin != NULL) num++;
+        // }
+        // if(num <= 2) g_object_set (G_OBJECT (this->pgie), "interval", 1, NULL);
+        // else if(num > 2 && num <= 4) g_object_set (G_OBJECT (this->pgie), "interval", 2, NULL);
+        // else if(num > 4 && num <= 6) g_object_set (G_OBJECT (this->pgie), "interval", 3, NULL);
+    }
 }

modules/userApp/src/user_app.cpp

@@ -4,7 +4,7 @@
  * @Autor: lishengyin
  * @Date: 2021-10-13 09:35:42
  * @LastEditors: lishengyin
- * @LastEditTime: 2022-07-13 08:47:54
+ * @LastEditTime: 2022-07-13 16:59:50
  */
 #include "user_app.h"
 
@@ -211,8 +211,11 @@ namespace gsd_ds
                     data.source_bin = this->m_Infer->add_sources(data.sourceId, data.uri);
                     this->m_InferInfo->DataSources.push_back(data);
                 }
+                
                 this->m_Infer->RestartTask();
+
             }
+            this->m_Infer->DynamicAdjust();
             this->m_Infer->SetBatch(this->m_InferInfo->DataSources.size());
         }else{
             for(iter=this->m_InferInfo->DataSources.begin(); iter!=this->m_InferInfo->DataSources.end(); iter++){

source/config/config_infer_primary_yoloV5.txt

@@ -42,12 +42,12 @@ net-scale-factor=0.0039215697906911373
 model-color-format=0
 # custom-network-config=yolov3-tiny.cfg
 # model-file=yolov3-tiny.weights
-model-engine-file=./FP16/yolov5s.engine
+model-engine-file=./INT8/yolov5m.engine
 labelfile-path=labels.txt
 process-mode=1
 batch-size=12
 ## 0=FP32, 1=INT8, 2=FP16 mode
-network-mode=2
+network-mode=1
 num-detected-classes=80
 gie-unique-id=1
 network-type=0
@@ -55,7 +55,7 @@ network-type=0
 output-blob-names=prob
 ## 0=Group Rectangles, 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering)
 cluster-mode=2
-interval=0
+interval=1
 maintain-aspect-ratio=1
 parse-bbox-func-name=NvDsInferParseCustomYoloV5
 custom-lib-path=../../lib/libnvdsinfer_custom_impl_Yolo.so