/************************************************************************* * Copyright (C) [2021] by Cambricon, Inc. All rights reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. *************************************************************************/ #include #include #include #include "opencv2/highgui/highgui.hpp" #include "opencv2/imgproc/imgproc.hpp" #if (CV_MAJOR_VERSION >= 3) #include "opencv2/imgcodecs/imgcodecs.hpp" #endif #include "easyinfer/mlu_memory_op.h" #include "cnstream_logging.hpp" #include "inferencer2.hpp" #include "postproc.hpp" #include "video_preproc.hpp" #include "test_base.hpp" namespace cnstream { class FakeVideoPostproc : public cnstream::VideoPostproc { public: bool Execute(infer_server::InferData* result, const infer_server::ModelIO& output, const infer_server::ModelInfo& model) override { return true; } private: DECLARE_REFLEX_OBJECT_EX(FakeVideoPostproc, cnstream::VideoPostproc); }; // class FakeVideoPostproc IMPLEMENT_REFLEX_OBJECT_EX(FakeVideoPostproc, cnstream::VideoPostproc); class FakeVideoPreproc : public cnstream::VideoPreproc { public: bool Execute(infer_server::ModelIO* model_input, const infer_server::InferData& input_data, const infer_server::ModelInfo& model_info) { return true; } private: DECLARE_REFLEX_OBJECT_EX(FakeVideoPreproc, cnstream::VideoPreproc); }; // class FakeVideoPreproc IMPLEMENT_REFLEX_OBJECT_EX(FakeVideoPreproc, cnstream::VideoPreproc); static std::string GetModelPath() { edk::MluContext ctx; edk::CoreVersion core_ver = ctx.GetCoreVersion(); std::string model_path = ""; switch (core_ver) { case edk::CoreVersion::MLU220: model_path = "../../data/models/yolov3_b4c4_argb_mlu220.cambricon"; break; case edk::CoreVersion::MLU270: default: model_path = "../../data/models/yolov3_b4c4_argb_mlu270.cambricon"; break; } return model_path; } static std::string GetModelPathMM() { return "../../data/models/yolov3_nhwc.model"; } // the data is related to model static cnstream::CNFrameInfoPtr CreatData(std::string device_id, bool is_eos = false, bool mlu_data = true) { auto data = cnstream::CNFrameInfo::Create(device_id, is_eos); cv::Mat image = cv::imread(GetExePath() + "../../data/images/0.jpg"); int width = image.cols; int height = image.rows; size_t nbytes = width * height * sizeof(uint8_t) * 3; data->stream_id = "1"; std::shared_ptr frame(new (std::nothrow) CNDataFrame()); if (mlu_data) { void* frame_data = image.data; void* planes[CN_MAX_PLANES] = {nullptr, nullptr}; edk::MluMemoryOp mem_op; frame_data = mem_op.AllocMlu(nbytes); planes[0] = frame_data; // y plane planes[1] = reinterpret_cast(reinterpret_cast(frame_data) + width * height); // uv plane void* ptr_mlu[2] = {planes[0], planes[1]}; frame->ctx.dev_type = DevContext::DevType::MLU; frame->ctx.ddr_channel = std::stoi(device_id); frame->ctx.dev_id = std::stoi(device_id); frame->fmt = CNDataFormat::CN_PIXEL_FORMAT_YUV420_NV12; frame->dst_device_id = std::stoi(device_id); frame->frame_id = 1; data->timestamp = 1000; frame->width = width; frame->height = height; frame->stride[0] = frame->stride[1] = width; frame->CopyToSyncMem(ptr_mlu, true); std::shared_ptr objs(new (std::nothrow) CNInferObjs()); data->collection.Add(kCNDataFrameTag, frame); data->collection.Add(kCNInferObjsTag, objs); return data; } else { frame->frame_id = 1; data->timestamp = 1000; frame->width = width; frame->height = height; void* ptr_cpu[2] = {image.data, image.data + nbytes * 2 / 3}; frame->stride[0] = frame->stride[1] = width; frame->fmt = CNDataFormat::CN_PIXEL_FORMAT_YUV420_NV12; frame->ctx.dev_type = DevContext::DevType::CPU; frame->dst_device_id = std::stoi(device_id); frame->ctx.dev_id = std::stoi(device_id); frame->CopyToSyncMem(ptr_cpu, true); std::shared_ptr objs(new (std::nothrow) CNInferObjs()); data->collection.Add(kCNDataFrameTag, frame); data->collection.Add(kCNInferObjsTag, objs); return data; } return nullptr; } TEST(Inferencer2, Open) { bool use_magicmind = infer_server::Predictor::Backend() == "magicmind"; std::string exe_path = GetExePath(); std::string infer_name = "detector"; { // open success but param is lack std::unique_ptr infer(new Inferencer2(infer_name)); ModuleParamSet param; if (use_magicmind) { param["model_path"] = exe_path + GetModelPathMM(); param["model_input_pixel_format"] = "RGB24"; } else { param["model_path"] = exe_path + GetModelPath(); param["model_input_pixel_format"] = "ARGB32"; } param["preproc_name"] = "VideoPreprocCpu"; param["postproc_name"] = "VideoPostprocSsd"; EXPECT_TRUE(infer->Open(param)); } { // param is empty std::unique_ptr infer(new Inferencer2(infer_name)); ModuleParamSet param; EXPECT_FALSE(infer->Open(param)); } { // param is no registered. std::unique_ptr infer(new Inferencer2(infer_name)); ModuleParamSet param; if (use_magicmind) { param["model_path"] = exe_path + GetModelPathMM(); } else { param["model_path"] = exe_path + GetModelPath(); } param["preproc_name"] = "VideoPreprocCpu"; param["postproc_name"] = "VideoPostprocSsd"; param["no_such_key"] = "key"; EXPECT_FALSE(infer->Open(param)); } { // preproc_name is error std::unique_ptr infer(new Inferencer2(infer_name)); ModuleParamSet param; if (use_magicmind) { param["model_path"] = exe_path + GetModelPathMM(); } else { param["model_path"] = exe_path + GetModelPath(); } param["preproc_name"] = "no_such_preproc_class"; param["postproc_name"] = "VideoPostprocSsd"; EXPECT_FALSE(infer->Open(param)); } { // postproc_name is error std::unique_ptr infer(new Inferencer2(infer_name)); ModuleParamSet param; if (use_magicmind) { param["model_path"] = exe_path + GetModelPathMM(); param["preproc_name"] = "CNCV"; } else { param["model_path"] = exe_path + GetModelPath(); param["preproc_name"] = "RCOP"; } param["postproc_name"] = "no_such_postproc_name"; EXPECT_FALSE(infer->Open(param)); } { // postproc_name is empty std::unique_ptr infer(new Inferencer2(infer_name)); ModuleParamSet param; if (use_magicmind) { param["model_path"] = exe_path + GetModelPathMM(); param["preproc_name"] = "CNCV"; } else { param["model_path"] = exe_path + GetModelPath(); param["preproc_name"] = "RCOP"; } param["postproc_name"] = ""; EXPECT_FALSE(infer->Open(param)); } { // model_path is error std::unique_ptr infer(new Inferencer2(infer_name)); ModuleParamSet param; if (use_magicmind) { param["model_path"] = "/home/no.model"; param["preproc_name"] = "CNCV"; } else { param["model_path"] = "/home/error_path"; param["preproc_name"] = "RCOP"; } param["postproc_name"] = "VideoPostprocSsd"; EXPECT_FALSE(infer->Open(param)); // check model path in inference, infer_handler no check } } TEST(Inferencer2, Process) { bool use_magicmind = infer_server::Predictor::Backend() == "magicmind"; std::string exe_path = GetExePath(); std::string infer_name = "detector"; std::unique_ptr infer(new Inferencer2(infer_name)); ModuleParamSet param; if (use_magicmind) { param["model_path"] = exe_path + GetModelPathMM(); param["model_input_pixel_format"] = "RGB24"; } else { param["model_path"] = exe_path + GetModelPath(); param["model_input_pixel_format"] = "ARGB32"; } param["preproc_name"] = "FakeVideoPreproc"; param["postproc_name"] = "FakeVideoPostproc"; param["device_id"] = "0"; { // CNFrameInfo empty ASSERT_TRUE(infer->Open(param)); std::shared_ptr data = nullptr; EXPECT_EQ(infer->Process(data), -1); } { // CNFrameInfo is eos ASSERT_TRUE(infer->Open(param)); std::string device_id = param["device_id"]; bool is_eos = true; auto data = CreatData(device_id, is_eos); EXPECT_EQ(infer->Process(data), 0); } { // CNFrameInfo no eos and data in mlu ASSERT_TRUE(infer->Open(param)); std::string device_id = param["device_id"]; bool is_eos = false; bool mlu_data = true; auto data = CreatData(device_id, is_eos, mlu_data); EXPECT_EQ(infer->Process(data), 0); } { // CNFrameInfo no eos and data in cpu ASSERT_TRUE(infer->Open(param)); std::string device_id = param["device_id"]; bool is_eos = false; bool mlu_data = false; auto data = CreatData(device_id, is_eos, mlu_data); EXPECT_EQ(infer->Process(data), 0); } EXPECT_NO_THROW(infer->Close()); } } // namespace cnstream