要提交的变更：

	修改：     modules/CMakeLists.txt
	修改：     modules/InfineFilter/include/InfineFilter.h
	修改：     modules/InfineFilter/src/InfineFilter.cpp
	修改：     modules/recorder/include/recorder.hpp
	修改：     modules/recorder/src/recorder.cpp
	修改：     modules/recorder/src/video.cpp
	修改：     source/cns_launcher/configs/InfineFilter.json
	修改：     source/cns_launcher/configs/yolov5_object_detection_mlu220.json
	修改：     source/cns_launcher/object_tracking/config_template.json
	修改：     source/cns_launcher/object_tracking/detection_config.json
	修改：     source/cns_launcher/object_tracking/detection_test.json
	删除：     source/cns_launcher/object_tracking/output/output_stream_0.mp4
	修改：     source/cns_launcher/object_tracking/start.sh
	修改：     source/cns_launcher/object_tracking/test.sh
	修改：     source/common/postprocess/postprocess_yolov3.cpp
	修改：     source/files.list_video

modules/CMakeLists.txt

@@ -202,8 +202,6 @@ if(build_kafka)
 endif()
 
 
-
-
 set(module_list "")
 if(build_encode)
   list(APPEND module_list encode)

modules/InfineFilter/include/InfineFilter.h

@@ -4,7 +4,7 @@
  * @Autor: lishengyin
  * @Date: 2022-03-22 14:00:35
  * @LastEditors: lishengyin
- * @LastEditTime: 2022-06-28 09:54:59
+ * @LastEditTime: 2022-08-01 16:34:15
  */
 #ifndef __INFINEFILTER_H_
 #define __INFINEFILTER_H_
@@ -62,10 +62,14 @@ namespace cnstream{
 
         int Process(std::shared_ptr<cnstream::CNFrameInfo> data) override;
 
-        bool filter(cv::Mat image, std::shared_ptr<cnstream::CNInferObject>& object);
+        bool filter(cv::Mat& image, CNInferObjsPtr objs_holder);
 
-        bool AlienFilter(cv::Mat image, CNInferObjsPtr& objs_holder);
+        bool AlienFilter(CNInferObjsPtr& objs_holder);
+
+        bool proportionalFilter(cv::Mat& image, CNInferObjsPtr objs_holder);
         
+        bool nightFilter(cv::Mat& image, CNInferObjsPtr objs_holder);
+
         bool getNight();
 
         std::string getTime();
@@ -76,11 +80,12 @@ namespace cnstream{
         
         vector<std::string> labels;
         vector<std::string> AlienLabels;
-
+        
         bool saveFilterResult = true;
         bool Night_Filter = true;
         std::string files = "./output/";
         double Proportion_th = 0.2;
+        double Night_th = 0.2;
         int start = 21;
         int end = 6;
     };

modules/InfineFilter/src/InfineFilter.cpp

@@ -4,15 +4,15 @@
  * @Autor: lishengyin
  * @Date: 2022-03-22 14:00:46
  * @LastEditors: lishengyin
- * @LastEditTime: 2022-07-21 14:17:36
+ * @LastEditTime: 2022-08-31 15:58:56
  */
 #include "InfineFilter.h"
 
 namespace cnstream{
 
     bool InfineFilter::Open(cnstream::ModuleParamSet paramSet){
-        if (paramSet.find("FilterId") != paramSet.end()) {
-            std::string json = paramSet["FilterId"];
+        if (paramSet.find("labels") != paramSet.end()) {
+            std::string json = paramSet["labels"];
             rapidjson::Document doc;
             if (doc.Parse<rapidjson::kParseCommentsFlag>(json.c_str()).HasParseError()) {
                 return false;
@@ -79,6 +79,11 @@ namespace cnstream{
         return true;
     }
 
+    /**
+     * @description: Process
+     * @param {shared_ptr<cnstream::CNFrameInfo>} data
+     * @return {*}
+     */    
     int InfineFilter::Process(std::shared_ptr<cnstream::CNFrameInfo> data){
         CNDataFramePtr frame = data->collection.Get<CNDataFramePtr>(kCNDataFrameTag);
         if (frame->width < 0 || frame->height < 0) {
@@ -95,138 +100,151 @@ namespace cnstream{
             this->TransmitData(data);
             return 0;
         }
+        cv::Mat img = frame->ImageBGR();
+        filter(img, objs_holder);
 
-        // 异类过滤器
-        this->AlienFilter(frame->ImageBGR(), objs_holder);
+        return 0;
+    }
 
-        // 过滤某些Id
-        for(auto iter = objs_holder->objs_.begin(); iter != objs_holder->objs_.end(); ){
-            std::shared_ptr<cnstream::CNInferObject> object = *iter;
-            if (!object) continue;
-            auto it = find(this->labels.begin(), this->labels.end(), object->id);
-            if(it != this->labels.end()){
-                iter = objs_holder->objs_.erase(iter);
-            }else{
-                iter++;
-            }
+    /**
+     * @description: 过滤器  过滤器优先级： 比例过滤器 > 异类过滤器 > 夜间过滤器
+     * @param {Mat&} image
+     * @param {CNInferObjsPtr} objs_holder
+     * @return {*}
+     */        
+    bool InfineFilter::filter(cv::Mat& image, CNInferObjsPtr objs_holder){
+        
+        // 比例过滤器
+        if(this->proportionalFilter(image, objs_holder) == false) {
+            objs_holder->objs_.clear();
         }
 
-        if(!this->Night_Filter) return 0;
-        if(!getNight()) return 0;
-        
-        // 过滤星星、月亮、飞机
+        // 异类过滤器
+        this->AlienFilter(objs_holder);
+
+        // 夜间过滤器
+        this->nightFilter(image, objs_holder);
+
+        return true;
+    }
+
+    /**
+     * @description: 异类过滤器 labels以外的定义为异物
+     * @param {CNInferObjsPtr&} objs_holder
+     * @return {*}
+     */         
+    bool InfineFilter::AlienFilter(CNInferObjsPtr& objs_holder){
+        if(objs_holder->objs_.size() == 0 || labels.empty()) return false;
         for(auto iter = objs_holder->objs_.begin(); iter != objs_holder->objs_.end();){
             std::shared_ptr<cnstream::CNInferObject> object = *iter;
-            if (!object) continue;
-            if(!this->filter(frame->ImageBGR(), object)){
-                iter = objs_holder->objs_.erase(iter);
-            }else{
-                iter++;
-            }
-        }   
-        return 0;
+            if(!object) continue;
+            auto it = find(labels.begin(), labels.end(),object->id);
+            if(it == this->labels.end()) objs_holder->objs_.erase(iter);
+            else iter++;
+        }
+        return true;
     }
 
     /**
-     * @description: 过滤器
-     * @param {Mat} image
+     * @description: 比例过滤器
+     * @param {Mat&} image
+     * @param {CNInferObjsPtr} objs_holder
      * @return {*}
      */    
-    bool InfineFilter::filter(cv::Mat image, std::shared_ptr<cnstream::CNInferObject>& object){
-        cv::Mat img = image.clone();
-        // 腐蚀
-        erode(img, img, cv::Mat());
-
-        cv::Rect rect = cv::Rect(object->bbox.x * image.cols,object->bbox.y * image.rows, object->bbox.w * image.cols, object->bbox.h * image.rows);
-        cv::Mat dst = img(rect);
-        cv::Mat imgThresholded;
-        // 转换为二值图
-        cv::inRange(dst, cv::Scalar(128, 128, 128), cv::Scalar(255, 255, 255), imgThresholded);
-        cv::threshold(imgThresholded, imgThresholded, 1, 255, cv::THRESH_BINARY);
-        
-        // 寻找轮廓
-        vector<vector<cv::Point>> contours;
-        vector<cv::Vec4i> hierarchy;
-        cv::findContours(imgThresholded, contours, hierarchy, cv::RETR_TREE, cv::CHAIN_APPROX_SIMPLE);
-
-        if(contours.empty()) return true;
-        // 获取最大轮廓
-        double max_area = cv::contourArea(cv::InputArray(contours[0]), false);
-        for(unsigned long long i = 1; i < contours.size(); i++){
-            double temp_area = cv::contourArea(cv::InputArray(contours[i]), false);
-            if(max_area < temp_area){
-                max_area = temp_area;
+    bool InfineFilter::proportionalFilter(cv::Mat& image, CNInferObjsPtr objs_holder){
+        int target_num = 0, Alien_num = 0;
+        if(objs_holder->objs_.empty() || AlienLabels.empty() || labels.empty()) return true;
+        for(auto iter = objs_holder->objs_.begin(); iter != objs_holder->objs_.end(); iter++){
+            std::shared_ptr<cnstream::CNInferObject> object = *iter;
+            auto it = find(labels.begin(), labels.end(), object->id);
+            if(it != this->labels.end()){
+                target_num++;
+                continue;
+            } 
+            it = find(AlienLabels.begin(), AlienLabels.end(), object->id);
+            if(it != this->AlienLabels.end()){
+                Alien_num++;
             }
         }
-        double b = max_area / (object->bbox.w * image.cols * object->bbox.h * image.rows);
-        // 计算轮廓比例
-        if(b > 0.5){
-            if(saveFilterResult){
+        double Proportion = (double)Alien_num / (double)(target_num + Alien_num);
+        if(Proportion < Proportion_th) return true;
+        if(this->saveFilterResult){
+            cv::Mat img = image.clone();
+            for(auto iter = objs_holder->objs_.begin(); iter != objs_holder->objs_.end(); iter++){
+                std::shared_ptr<cnstream::CNInferObject> object = *iter;
                 cv::Point top_left;
                 cv::Point bottom_right;
                 top_left.x = object->bbox.x * image.cols;
                 top_left.y = object->bbox.y * image.rows;
                 bottom_right.x = top_left.x + (object->bbox.w * image.cols);
                 bottom_right.y = top_left.y + (object->bbox.h * image.rows);
-                cv::Point logo_pos(5, image.rows - 5);
+                cv::Point logo_pos(top_left.x - 5, bottom_right.y- 5);
                 cv::Scalar color(200, 200, 200);
-                cv::putText(img, std::to_string(b), logo_pos, 0, 1, color, 2);
+                cv::putText(img, object->id, logo_pos, 0, 1, color, 2);
                 cv::rectangle(img, top_left, bottom_right, cv::Scalar(0, 255, 0), 4);
-                if(opendir(this->files.c_str()) == NULL){
-                    mkdir((this->files).c_str(),S_IRWXU|S_IRWXG|S_IRWXO);
-                } 
-                cv::imwrite(this->files + getTime() + ".jpg", img);
             }
-            return false;
+            cv::resize(img, img, cv::Size(960, 540));
+            std::string Dir = this->files + "/proportionalFilter/";
+            if(opendir(Dir.c_str()) == NULL){
+                mkdir((Dir).c_str(),S_IRWXU|S_IRWXG|S_IRWXO);
+            } 
+            cv::imwrite(Dir + getTime() + ".jpg", img);
         }
-        return true;
+        return false;
     }
 
     /**
-     * @description: 异类过滤器
-     * @param {Mat} image
+     * @description: 夜间过滤器
+     * @param {Mat&} image
+     * @param {CNInferObjsPtr} objs_holder
      * @return {*}
-     */      
-    bool InfineFilter::AlienFilter(cv::Mat image, CNInferObjsPtr& objs_holder){
-        // 计算占比
-        int num = 0;
-        if(objs_holder->objs_.size() == 0) return true;
-        for(auto iter = objs_holder->objs_.begin(); iter != objs_holder->objs_.end(); ){
+     */    
+    bool InfineFilter::nightFilter(cv::Mat& image, CNInferObjsPtr objs_holder){
+        vector<CNInferBoundingBox> result;
+        if(objs_holder->objs_.size() == 0) return false;
+        if(!this->getNight()) return false;
+        cv::Mat img = image.clone();
+        cv::erode(img, img, cv::Mat());
+        for(auto iter = objs_holder->objs_.begin(); iter != objs_holder->objs_.end();){
             std::shared_ptr<cnstream::CNInferObject> object = *iter;
-            if (!object) continue;
-            auto it = find(this->AlienLabels.begin(), this->AlienLabels.end(), object->id);
-            if(it != this->AlienLabels.end()){
-                num++;
-            }
-        }
-        double Proportion = (double)num / (double)(objs_holder->objs_.size());
-        if(Proportion > this->Proportion_th) {
-            if(this->saveFilterResult){
-                cv::Mat img = image.clone();
-                for(auto iter = objs_holder->objs_.begin(); iter != objs_holder->objs_.end(); iter++){
-                    std::shared_ptr<cnstream::CNInferObject> object = *iter;
-                    if (!object) continue;
-                    cv::Point top_left;
-                    cv::Point bottom_right;
-                    top_left.x = object->bbox.x * image.cols;
-                    top_left.y = object->bbox.y * image.rows;
-                    bottom_right.x = top_left.x + (object->bbox.w * image.cols);
-                    bottom_right.y = top_left.y + (object->bbox.h * image.rows);
-                    cv::Point logo_pos(5, image.rows - 5);
-                    cv::Scalar color(200, 200, 200);
-                    cv::putText(img, object->id, logo_pos, 0, 1, color, 2);
-                    cv::rectangle(img, top_left, bottom_right, cv::Scalar(0, 255, 0), 4);
+            cv::Rect rect = cv::Rect(object->bbox.x * image.cols,object->bbox.y * image.rows, object->bbox.w * image.cols, object->bbox.h * image.rows);
+            cv::Mat dst = img(rect);
+            cv::Mat imgThresholded;
+            cv::inRange(dst, cv::Scalar(128, 128, 128), cv::Scalar(255, 255, 255), imgThresholded);
+            cv::threshold(imgThresholded, imgThresholded, 1, 255, cv::THRESH_BINARY);
+            vector<vector<cv::Point>> contours;
+            vector<cv::Vec4i> hierarchy;
+            cv::findContours(imgThresholded, contours, hierarchy, cv::RETR_TREE, cv::CHAIN_APPROX_SIMPLE);
+            if(contours.empty()) continue;
+            double max_area = cv::contourArea(cv::InputArray(contours[0]), false);
+            for(unsigned long long i = 1; i < contours.size(); i++){
+                double temp_area = cv::contourArea(cv::InputArray(contours[i]), false);
+                if(max_area < temp_area){
+                    max_area = temp_area;
                 }
-                if(opendir(this->files.c_str()) == NULL){
-                    mkdir((this->files).c_str(),S_IRWXU|S_IRWXG|S_IRWXO);
-                } 
-                cv::imwrite(this->files + getTime() + "Alien.jpg", img);
             }
+            double b = max_area / (object->bbox.w * image.cols * object->bbox.h * image.rows);
+            if(b > Night_th){
+                iter = objs_holder->objs_.erase(iter);
+                result.push_back(object->bbox);
+            }else iter++;
         }
+        if(result.empty() || !this->saveFilterResult) return true;
+        for(auto iter = result.begin(); iter != result.end(); iter++){
+            cv::Point top_left;
+            cv::Point bottom_right;
+            bottom_right.x = top_left.x + (iter->w * image.cols);
+            bottom_right.y = top_left.y + (iter->h * image.rows);
+            cv::rectangle(img, top_left, bottom_right, cv::Scalar(0, 255, 0), 4);
+        }
+        cv::resize(img, img, cv::Size(960, 540));
+        if(opendir(this->files.c_str()) == NULL){
+            mkdir((this->files).c_str(),S_IRWXU|S_IRWXG|S_IRWXO);
+        }
+        cv::imwrite(this->files + getTime() + ".jpg", img);
         return true;
     }
 
-
     /**
      * @description: 判断时间是否为晚上
      * @param {*}

modules/recorder/include/recorder.hpp

@@ -38,6 +38,8 @@ namespace cnstream{
             
             string m_enc_type;
             string m_outputDir;
+            
+            map<std::string, std::shared_ptr<video>> m_videos;
 
         public:
             explicit Recorder(const std::string &name);

modules/recorder/src/recorder.cpp

@@ -60,15 +60,15 @@ namespace cnstream{
             LOGE(Recorder) << "Frame width and height can not be lower than 0.";
             return -1;
         }
-        if (!data->IsEos()) {
-            if(m_Video == nullptr){
-                // 创造对象
-                m_Video = std::make_shared<video>(this->m_outputDir,this->m_enc_type,this->m_frame_rate, this->m_videoMaxTime,this->m_cacheTime, Size(this->m_dst_width, this->m_dst_height));
+        if (!data->IsEos()) {  
+            auto iter = m_videos.find(data->stream_id);
+            if(iter == m_videos.end() || m_videos[data->stream_id] == nullptr){
+                m_videos[data->stream_id] = std::make_shared<video>(this->m_outputDir,this->m_enc_type,this->m_frame_rate, this->m_videoMaxTime,this->m_cacheTime, Size(this->m_dst_width, this->m_dst_height));
             }
             //传输数据
-            if(m_Video -> capture(data)){
+            if(m_videos[data->stream_id] -> capture(data)){
                 //视频截取成功 释放资源
-                m_Video = nullptr;
+                m_videos[data->stream_id] = nullptr;
             }
         }
         return 0;

modules/recorder/src/video.cpp

@@ -36,11 +36,10 @@ bool video::capture(std::shared_ptr<cnstream::CNFrameInfo>& data)
     
     auto objs_holder = data->collection.Get<cnstream::CNInferObjsPtr>(cnstream::kCNInferObjsTag);
 
-
     // if (data->datas.find(CNObjsVecKey) != data->datas.end()) {
     //     input_objs = (cnstream::any_cast<CNObjsVec>(data->datas[CNObjsVecKey]));
     // }
-
+    
     Mat dst = frame->ImageBGR();
     cv::resize(dst, dst, this -> m_frameSize);
     if (objs_holder->objs_.size() == 0) 
@@ -75,7 +74,7 @@ bool video::capture(std::shared_ptr<cnstream::CNFrameInfo>& data)
             info = localtime(&rawtime);
             strftime(ctime, 80, "%Y-%m-%d_%H:%M:%S", info);
             string fileName = ctime;
-            this -> m_fileName =  fileName + this->m_enc_type;
+            this -> m_fileName =  fileName + "_" + data->stream_id + this->m_enc_type;
             this -> m_outputDir = this -> m_outputDir + fileName.substr(0,7) + "/";
             if(opendir(this -> m_outputDir.c_str()) == NULL){
                 mkdir((this->m_outputDir).c_str(),S_IRWXU|S_IRWXG|S_IRWXO);

source/cns_launcher/configs/InfineFilter.json

@@ -4,11 +4,11 @@
         "parallelism" : 1,
         "max_input_queue_size" : 20,
         "custom_params" : {
-            "FilterId" : {
-                "label":["1"]
+            "labels" : {
+                "label":["16"]
             },
             "AlienLabels":{
-                "label":["1", "2"]
+                "label":["2"]
             },
             "files": "./output/",
             "saveFilterResult" : "true",

source/cns_launcher/configs/yolov5_object_detection_mlu220.json

@@ -4,7 +4,7 @@
     "parallelism" : 2,
     "max_input_queue_size" : 20,
     "custom_params" : {
-      "model_path" : "../../../data/models/yolov5m_best6_b_b_int8_0627_4b_4c.cambricon",
+      "model_path" : "../../../data/models/yolov5_4c4b_rgb_220.cambricon",
       "func_name" : "subnet0",
       // Uncomment the following 2 lines to use cncv
       // "preproc_name" : "CNCV",

source/cns_launcher/object_tracking/config_template.json

@@ -6,14 +6,14 @@
 
   "subgraph:decode" : {
     "config_path" : "../configs/decode_config.json",
-    "next_modules" : ["subgraph:object_detection"]
-  },
-
-  "subgraph:object_detection" : {
-    "config_path" : "../configs/__NN___object_detection___PLATFORM_PLACEHOLDER__.json",
     "next_modules" : ["subgraph:udp"]
   },
 
+  // "subgraph:object_detection" : {
+  //   "config_path" : "../configs/__NN___object_detection___PLATFORM_PLACEHOLDER__.json",
+  //   "next_modules" : ["subgraph:udp"]
+  // },
+
   // "subgraph:object_tracking" : {
   //   "config_path" : "../configs/object_tracking___PLATFORM_PLACEHOLDER__.json",
   //   "next_modules" : ["subgraph:udp"]

source/cns_launcher/object_tracking/detection_config.json

@@ -11,7 +11,7 @@
 
   "subgraph:object_detection" : {
     "config_path" : "../configs/yolov5_object_detection_mlu220.json",
-    "next_modules" : ["subgraph:InfineFilter"]
+    "next_modules" : ["subgraph:udp"]
   },
 
   // "subgraph:object_tracking" : {
@@ -19,10 +19,10 @@
   //   "next_modules" : ["subgraph:udp"]
   // },
 
-  "subgraph:InfineFilter" : {
-    "config_path" : "../configs/InfineFilter.json",
-    "next_modules" : ["subgraph:udp"]
-  },
+  // "subgraph:InfineFilter" : {
+  //   "config_path" : "../configs/InfineFilter.json",
+  //   "next_modules" : ["subgraph:udp"]
+  // },
 
   "subgraph:udp" : {
     "config_path" : "../configs/udp.json",

source/cns_launcher/object_tracking/detection_test.json

@@ -11,19 +11,19 @@
   
     "subgraph:object_detection" : {
       "config_path" : "../configs/yolov5_object_detection_test.json",
-      "next_modules" : ["subgraph:InfineFilter"]
+      "next_modules" : ["subgraph:udp"]
     },
 
-    "subgraph:InfineFilter" : {
-      "config_path" : "../configs/InfineFilter.json",
-      "next_modules" : ["subgraph:object_tracking"]
-    },
+    // "subgraph:InfineFilter" : {
+    //   "config_path" : "../configs/InfineFilter.json",
+    //   "next_modules" : ["subgraph:object_tracking"]
+    // },
 
   
-    "subgraph:object_tracking" : {
-      "config_path" : "../configs/object_tracking_mlu220.json",
-      "next_modules" : ["subgraph:udp"]
-    },
+    // "subgraph:object_tracking" : {
+    //   "config_path" : "../configs/object_tracking_mlu220.json",
+    //   "next_modules" : ["subgraph:udp"]
+    // },
   
     "subgraph:udp" : {
       "config_path" : "../configs/udp.json",

source/cns_launcher/object_tracking/start.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-./run.sh mlu220 video yolov5
+./run.sh mlu220 rtsp yolov5

source/cns_launcher/object_tracking/test.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+while true
+do
 CURRENT_DIR=$(cd $(dirname ${BASH_SOURCE[0]});pwd)
 CNSTREAM_ROOT=${CURRENT_DIR}/../../..
 SOURCE_ROOT=${CNSTREAM_ROOT}/source
@@ -11,3 +13,6 @@ ${SOURCE_ROOT}/bin/main  \
     --src_frame_rate 25   \
     --config_fname ${CURRENT_DIR}/detection_test.json \
     --log_to_file=true
+done
+
+

source/common/postprocess/postprocess_yolov3.cpp

@@ -91,7 +91,7 @@ class PostprocYolov3 : public cnstream::Postproc {
       obj->bbox.w = std::min(1.0f - obj->bbox.x, right - left);
       obj->bbox.h = std::min(1.0f - obj->bbox.y, bottom - top);
 
-      if (obj->bbox.h <= 0 || obj->bbox.w <= 0 || (obj->score < threshold_ && threshold_ > 0) || obj->id != "14") continue;
+      if (obj->bbox.h <= 0 || obj->bbox.w <= 0 || (obj->score < threshold_ && threshold_ > 0)) continue;
       std::lock_guard<std::mutex> objs_mutex(objs_holder->mutex_);
       objs.push_back(obj);
     }

source/files.list_video

@@ -1 +1 @@
-/workspace/gsdv3.0/CNStream/data/videos/Bird.mp4
+rtsp://admin:a1234567@192.168.160.130:554/h264/ch1/main/av_stream