加入Cleaner模块

.vscode/settings.json

@@ -80,7 +80,8 @@
         "__tree": "cpp",
         "locale": "cpp",
         "stack": "cpp",
-        "__node_handle": "cpp"
+        "__node_handle": "cpp",
+        "__locale": "cpp"
     },
     "C_Cpp.errorSquiggles": "Disabled",
     "commentTranslate.targetLanguage": "en"

CMakeLists.txt

@@ -22,6 +22,7 @@ option(build_md5 "build module md5" ON)
 option(build_userApp "build module userApp" ON)
 option(build_TCPClient "build module TCPClient" ON)
 option(build_recorder "build module recorder" ON)
+option(build_Cleaner "build modele Cleaner" ON)
 option(WITH_FFMPEG "with ffmpeg" ON)
 option(WITH_OPENCV "with opencv" ON)
 option(WITH_ZLMediaKit "with ZLMediaKit" OFF)
@@ -156,6 +157,7 @@ include_directories(${PROJECT_SOURCE_DIR}/modules/TCPClient/include/)
 include_directories(${PROJECT_SOURCE_DIR}/modules/dataType/include/)
 include_directories(${PROJECT_SOURCE_DIR}/modules/HttpClient/include/)
 include_directories(${PROJECT_SOURCE_DIR}/modules/recorder/include/)
+include_directories(${PROJECT_SOURCE_DIR}/modules/Cleaner/include/)
 
 include_directories(${ToolKit_Root})
 include_directories(${MediaKit_Root})

modules/CMakeLists.txt

@@ -39,6 +39,10 @@ if(build_recorder)
   list(APPEND module_list recorder)
   install(DIRECTORY recorder/include/ DESTINATION include)
 endif()
+if(build_Cleaner)
+  list(APPEND module_list Cleaner)
+  install(DIRECTORY Cleaner/include/ DESTINATION include)
+endif()
 
 if(HAVE_FFMPEG)
   include_directories(${FFMPEG_INCLUDE_DIR})

modules/Cleaner/include/Cleaner.h

@@ -0,0 +1,74 @@
+#pragma once
+
+#include <iostream>
+#include <map>
+#include <time.h>
+#include <dirent.h>
+#include <time.h>
+#include <cstdlib>
+#include <dirent.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "Util/logger.h"
+#include "Util/NoticeCenter.h"
+#include "Poller/EventPoller.h"
+#include "Player/PlayerProxy.h"
+#include "Rtmp/RtmpPusher.h"
+#include "Common/config.h"
+#include "Pusher/MediaPusher.h"
+#include "Extension/Frame.h"
+#include "Util/SqlPool.h"
+#include "Network/TcpClient.h"
+#include "Poller/Timer.h"
+// opencv
+#include <opencv2/core.hpp>
+#include <opencv2/videoio.hpp>
+#include <opencv2/highgui.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/imgproc/types_c.h>
+#include "Util/logger.h"
+#include <cstdlib>
+#include "recorder.h"
+
+using namespace toolkit;
+using namespace std;
+
+
+namespace MIVA
+{
+    class Cleaner
+    {
+    public:
+        using Ptr = std::shared_ptr<Cleaner>;
+        int ClearType = 1;  // 清理类型  0   - 不清除记录
+                            //           1   - 超过设定阈值就删除第一个,保持不超过阈值
+                            //           2   - 超过设定阈值，直接删除全部记录
+                            //           3   - 设定时间，如只保留前多少小时的数据
+
+        int recordMax = 100;
+ 
+        int validTime = 1; // 有效时间 时间单位 天
+    public:
+        Cleaner();
+        ~Cleaner();
+
+        /**
+         * @description: 单例
+         * @param {*}
+         * @return {*}
+         */ 
+        static std::shared_ptr<Cleaner> CreateNew();
+
+        /**
+         * @description: 清除历史记录
+         * @param {*}
+         * @return {*}
+         */        
+        int32_t ClearHistory();
+    protected:
+        int rmDir(std::string file_name);
+    };
+
+} // namespace MIVA

modules/Cleaner/src/Cleaner.cpp

@@ -0,0 +1,109 @@
+#include "Cleaner.h"
+
+namespace MIVA
+{
+    std::shared_ptr<Cleaner> m_Cleaner = nullptr;
+    Cleaner::Cleaner()
+    {
+
+    }
+
+    Cleaner::~Cleaner()
+    {
+
+    }
+
+    /**
+     * @description: 单例
+     * @param {*}
+     * @return {*}
+     */    
+    std::shared_ptr<Cleaner> Cleaner::CreateNew()
+    {
+        if(m_Cleaner == nullptr) m_Cleaner = std::make_shared<Cleaner>();
+        return m_Cleaner;
+    }
+
+    /**
+     * @description: 清除历史几率
+     * @param {*}
+     * @return {*}
+     */    
+    int32_t Cleaner::ClearHistory()
+    {
+        vector<vector<string>> sqlRet;
+        if(!this->ClearType) return -1;
+
+        if(this->ClearType == 1){  // 超过设定阈值就删除第一个,保持不超过阈值
+            SqlWriter sqlSelect("SELECT Id,time,OutPath FROM MIVA_DB.`InferRecord` WHERE finish = 1 AND Del = 0 ORDER BY Id ASC");
+            sqlSelect << sqlRet;
+            if(sqlRet.empty()) return -1;
+            if(sqlRet.size() > this->recordMax){
+                int count = sqlRet.size() - this->recordMax;
+                auto iter = sqlRet.begin();
+                for(int i = 0; i < count; i++){
+                    vector<vector<string>> sqlRet1;
+                    std::string shell = "rm -rf " + (*iter)[2];
+                    system(shell.c_str());
+                    DebugL << shell << endl;
+                    SqlWriter Sqlupdata("UPDATE MIVA_DB.InferRecord SET Del = 1 where Id = '?'");
+                    Sqlupdata << std::atoi((*iter)[0].c_str()) << sqlRet1;
+                    iter++;
+                }
+            }
+        }else if(this->ClearType == 2){ // 超过设定阈值，直接删除全部记录
+            SqlWriter sqlSelect("SELECT Id,time,OutPath FROM MIVA_DB.`InferRecord` WHERE finish = 1 AND Del = 0 ORDER BY Id ASC");
+            sqlSelect << sqlRet;
+            if(sqlRet.size() > this->recordMax){
+                recorder::Ptr m_recorder = recorder::CreateNew();
+                std::string shell = "rm -rf " + m_recorder->Dir + "/*";
+                system(shell.c_str());
+                DebugL << shell << endl;
+                vector<vector<string>> sqlRet1;
+                SqlWriter Sqlupdata("UPDATE MIVA_DB.InferRecord SET Del = 1");
+                Sqlupdata << sqlRet1;
+            }
+        }else if(this->ClearType == 3){ // 设定时间，如只保留前多少小时的数据
+            SqlWriter sqlSelect("SELECT Id,time,OutPath FROM MIVA_DB.`InferRecord` WHERE DATE(time) <= DATE(DATE_SUB(NOW(),INTERVAL '?' day))");
+            sqlSelect << this-> validTime << sqlRet;
+            for (auto& line : sqlRet){
+                std::string shell = "rm -rf " + line[2];
+                system(shell.c_str());
+                DebugL << shell << endl;
+            }
+            vector<vector<string>> sqlRet1;
+            SqlWriter sqlUpdate("UPDATE MIVA_DB.InferRecord SET Del = 1 where DATE(time) <= DATE(DATE_SUB(NOW(),INTERVAL '?' day));");
+            sqlUpdate << this-> validTime << sqlRet1;
+        }
+        return 0;
+    }
+
+    int Cleaner::rmDir(std::string file_name)
+    {
+        std::string file_path = file_name;
+        struct stat st;    
+        if(lstat(file_path.c_str(),&st) == -1)
+        {
+            return -1;
+        }
+        if(S_ISREG(st.st_mode))
+        {
+            if(unlink(file_path.c_str()) == -1)
+            {
+                return -1;
+            }    
+        }
+        else if(S_ISDIR(st.st_mode))
+        {
+            if(file_name == "." || file_name == "..")
+            {
+                return -1;
+            }    
+            if(rmDir(file_path) == -1)//delete all the files in dir.
+            {
+                return -1;
+            }
+        }
+        return 0;
+    }
+} // namespace MIVA

modules/recorder/src/recorder.cpp

@@ -27,19 +27,12 @@ namespace MIVA
         auto iter = this->videoWriters.find(Pid);
         if(iter != this->videoWriters.end()){
             if( !outType || iter->second.fileName == ""){
-                time_t rawtime;
-                char ctime[80];
-                struct tm *info;
-                time(&rawtime);
-                info = localtime(&rawtime);
-                strftime(ctime, 80, "%Y-%m-%d_%H:%M:%S", info);
                 string fileName = this->outDir;
-                string time1 = ctime;
                 if(outType){
-                    fileName += time1 + "_" + Pid + ".mp4";
+                    fileName +=  Pid + ".mp4";
                     iter->second.videoWriter = std::make_shared<cv::VideoWriter>("appsrc ! autovideoconvert ! omxh264enc  ! matroskamux ! filesink location=" + fileName + " sync=false", 0, (double)2, cv::Size(1920, 1080), true);
                 }else{
-                    fileName += time1 + "_" + Pid + ".jpg";
+                    fileName +=  Pid + ".jpg";
                     cv::imwrite(fileName, frame);
                     iter->second.videoWriter = nullptr;
                 }
@@ -48,20 +41,12 @@ namespace MIVA
         }
         else{
             Writer writer;
-
-            time_t rawtime;
-            char ctime[80];
-            struct tm *info;
-            time(&rawtime);
-            info = localtime(&rawtime);
-            strftime(ctime, 80, "%Y-%m-%d_%H:%M:%S", info);
             string fileName = this->outDir;
-            string time1 = ctime;
             if(outType){
-                fileName += time1 + "_" + Pid + ".mp4";
+                fileName += Pid + ".mp4";
                 writer.videoWriter = std::make_shared<cv::VideoWriter>("appsrc ! autovideoconvert ! omxh264enc  ! matroskamux ! filesink location=" + fileName + " sync=false", 0, (double)2, cv::Size(1920, 1080), true);
             }else{
-                fileName += time1 + "_" + Pid + ".jpg";
+                fileName +=  Pid + ".jpg";
                 cv::imwrite(fileName, frame);
                 writer.videoWriter = nullptr;
             }

modules/userApp/include/user_app.h

@@ -4,7 +4,7 @@
  * @Autor: lishengyin
  * @Date: 2021-10-13 09:41:50
  * @LastEditors: lishengyin
- * @LastEditTime: 2022-01-06 15:24:42
+ * @LastEditTime: 2022-01-10 08:42:37
  */
 #pragma once
 
@@ -51,6 +51,7 @@
 #include "Notices.h"
 #include "HttpClient.h"
 #include "recorder.h"
+#include "Cleaner.h"
 #include <httplib.h>
 
 using namespace toolkit;
@@ -134,6 +135,11 @@ namespace MIVA
          */        
         static uint8_t GradeDetermination(int num);
         
+        /**
+         * @description: 
+         * @param {string} Pid
+         * @return {*}
+         */        
         static int GetAddrNum(std::string Pid);
     private:
         IniFile m_ini;
@@ -172,6 +178,7 @@ namespace MIVA
         std::shared_ptr<Inference> m_Infer = NULL;
         std::shared_ptr<HttpClient> m_httpClient = NULL;
         std::shared_ptr<recorder> m_recorder = nullptr;
+        std::shared_ptr<Cleaner> m_Cleaner = nullptr;
 
         Socket::Ptr m_udpClient = NULL;
 

modules/userApp/src/user_app.cpp

@@ -4,7 +4,7 @@
  * @Autor: lishengyin
  * @Date: 2021-10-13 09:35:42
  * @LastEditors: lishengyin
- * @LastEditTime: 2022-01-07 09:52:22
+ * @LastEditTime: 2022-01-10 10:05:32
  */
 #include "user_app.h"
 
@@ -91,8 +91,9 @@ namespace MIVA
         vector<vector<string>> sqlRet;
         
         this->m_recorder = recorder::CreateNew();
+        this->m_Cleaner = Cleaner::CreateNew();
         // 查询基础配置
-        SqlWriter sqlSelectConfig("SELECT account,outType,outPath,outEnable FROM MIVA_DB.`MivaConfig`");
+        SqlWriter sqlSelectConfig("SELECT account,outType,outPath,outEnable,ClearType,recordMax,validTime FROM MIVA_DB.`MivaConfig`");
         sqlSelectConfig << sqlRet;
         for(auto &line : sqlRet)
         {
@@ -100,6 +101,9 @@ namespace MIVA
             this->m_recorder->outType = std::atoi(line[1].c_str());
             this->m_recorder->Dir = line[2];
             this->m_recorder->RecordEnable = std::atoi(line[3].c_str()) ? true:false;
+            this->m_Cleaner->ClearType = std::atoi(line[4].c_str());
+            this->m_Cleaner->recordMax = std::atoi(line[5].c_str());
+            this->m_Cleaner->validTime = std::atoi(line[6].c_str());
         }
 
         // 链接Netty后端
@@ -113,8 +117,9 @@ namespace MIVA
         this->m_udpClient->bindUdpSock(this->PIS_port);
 
          // 清洗旧数据
-        SqlWriter sqlTruncate("truncate table MIVA_DB.InferTime");
-        sqlTruncate << sqlRet;
+        // SqlWriter sqlTruncate("truncate table MIVA_DB.InferRecord");
+        // sqlTruncate << sqlRet;
+        
         SqlWriter sqlUpdata("UPDATE MIVA_DB.DataSources SET Num=0,finish=0");
 
         sqlUpdata << sqlRet;
@@ -123,6 +128,7 @@ namespace MIVA
         sqlSelect << sqlRet;
 
         this->m_InferInfo = InferInfo::CreateNew();
+        
 
         // 初始化Deepstream
         m_Infer = std::make_shared<Inference>();
@@ -292,6 +298,8 @@ namespace MIVA
             this->ListenInferData();
             this->m_Infer->Recorder();
             this->m_Infer->StopTask();
+            // 清除历史
+            if(m_recorder->RecordEnable) this->m_Cleaner->ClearHistory();
         }
     }    
 
@@ -412,7 +420,7 @@ namespace MIVA
             passengerFlow.crType = atoi(line[1].c_str());
             passengerFlow.personNum = atoi(line[2].c_str());
             passengerFlow.detectionTime = line[3];
-
+            passengerFlow.detectionVideo = this->m_recorder->GetFileName(carCode);
             vector<vector<std::string>> sqlRet1;
             SqlWriter sqlSelect1("SELECT Uri,IP FROM MIVA_DB.`DataSources` WHERE Play=1 and Del=0 and CarId='?'");
             sqlSelect1 << atoi(line[4].c_str()) << sqlRet1;
@@ -544,7 +552,7 @@ namespace MIVA
         vector<vector<string>> sqlRet;
         vector<DataSource>::iterator iter;
 
-        SqlWriter sqlSelectConfig("SELECT account,outType,outPath,outEnable FROM MIVA_DB.`MivaConfig`");
+        SqlWriter sqlSelectConfig("SELECT account,outType,outPath,outEnable,ClearType,recordMax,validTime FROM MIVA_DB.`MivaConfig`");
         sqlSelectConfig << sqlRet;
         for(auto &line : sqlRet)
         {
@@ -552,6 +560,9 @@ namespace MIVA
             this->m_recorder->outType = std::atoi(line[1].c_str());
             this->m_recorder->Dir = line[2];
             this->m_recorder->RecordEnable = std::atoi(line[3].c_str()) ? true:false;
+            this->m_Cleaner->ClearType = std::atoi(line[4].c_str());
+            this->m_Cleaner->recordMax = std::atoi(line[5].c_str());
+            this->m_Cleaner->validTime = std::atoi(line[6].c_str());
         }
         
         if(this->m_Infer->enable == false || this->m_InferInfo->DataSources.empty()){
@@ -754,10 +765,24 @@ namespace MIVA
             num++;
             return;
         }
-        SqlWriter sqlInstall("INSERT INTO MIVA_DB.InferTime(time,InferTime) VALUES('?','?');");
+
+        SqlWriter sqlInstall("INSERT INTO MIVA_DB.InferRecord(time,InferTime,OutPath,batch_size,out_size,Finish,Del) VALUES('?','?','?','?','?','?','?');");
         char ctime[80];
         getDataTime(ctime);
-        sqlInstall << ctime << (time2.tv_sec - time1.tv_sec) *1000 + (time2.tv_nsec - time1.tv_nsec)/1000000 << sqlRet;
+        string outPath =  this->m_recorder->RecordEnable ? this->m_recorder->outDir : "";
+
+        int batch_size = 0;
+        for (auto iter = this->m_InferInfo->DataSources.begin(); iter != this->m_InferInfo->DataSources.end(); iter++){
+            if(iter->source_bin != NULL){
+                batch_size++;
+            }
+        }
+        int out_size = 0;
+        SqlWriter sqlSelectSize("SELECT * FROM MIVA_DB.`DataSources` WHERE Play = 1 and Del=0 and finish = 1");
+        sqlSelectSize << sqlRet;
+        out_size = sqlRet.size();
+        int finish = (batch_size == out_size) ? 1 : 0;
+        sqlInstall << ctime << (time2.tv_sec - time1.tv_sec) *1000 + (time2.tv_nsec - time1.tv_nsec)/1000000 << outPath << batch_size << out_size <<  finish  << 0 << sqlRet;
     }
     
     /**