From ee8f0d1163904e38f3a6117403af730068ab883a Mon Sep 17 00:00:00 2001 From: Tuan Nghia Nguyen Date: Thu, 14 May 2026 17:48:09 +1000 Subject: [PATCH] Suppot RTMPOSE classes --- ANSLIB/ANSLIB.h | 4 +- core/ANSLicensingSystem/ANSLicense.h | 4 +- modules/ANSODEngine/ANSONNXRTMPOSE.cpp | 673 ++++++++++++++++++++++++ modules/ANSODEngine/ANSONNXRTMPOSE.h | 159 ++++++ modules/ANSODEngine/ANSRTRTMPOSE.cpp | 702 +++++++++++++++++++++++++ modules/ANSODEngine/ANSRTRTMPOSE.h | 125 +++++ modules/ANSODEngine/CMakeLists.txt | 2 + modules/ANSODEngine/dllmain.cpp | 76 ++- 8 files changed, 1742 insertions(+), 3 deletions(-) create mode 100644 modules/ANSODEngine/ANSONNXRTMPOSE.cpp create mode 100644 modules/ANSODEngine/ANSONNXRTMPOSE.h create mode 100644 modules/ANSODEngine/ANSRTRTMPOSE.cpp create mode 100644 modules/ANSODEngine/ANSRTRTMPOSE.h diff --git a/ANSLIB/ANSLIB.h b/ANSLIB/ANSLIB.h index 245f93d..35e0cab 100644 --- a/ANSLIB/ANSLIB.h +++ b/ANSLIB/ANSLIB.h @@ -50,7 +50,9 @@ enum ModelType { ONNXSAM3 = 28, RTSAM3 = 29, ONNXYOLO = 30, - RTYOLO = 31 + RTYOLO = 31, + ONNXRTMPOSE=32, + RTRTMPOSE=33 }; enum TrackerType { BYTETRACK = 0, diff --git a/core/ANSLicensingSystem/ANSLicense.h b/core/ANSLicensingSystem/ANSLicense.h index 6a9cbd8..b18bc8f 100644 --- a/core/ANSLicensingSystem/ANSLicense.h +++ b/core/ANSLicensingSystem/ANSLicense.h @@ -241,7 +241,9 @@ namespace ANSCENTER ONNXSAM3=28, RTSAM3 = 29, ONNXYOLO = 30, - RTYOLO = 31 + RTYOLO = 31, + ONNXRTMPOSE=32, + RTRTMPOSE=33 }; enum TrackerType { BYTETRACK = 0, diff --git a/modules/ANSODEngine/ANSONNXRTMPOSE.cpp b/modules/ANSODEngine/ANSONNXRTMPOSE.cpp new file mode 100644 index 0000000..e69c71a --- /dev/null +++ b/modules/ANSODEngine/ANSONNXRTMPOSE.cpp @@ -0,0 +1,673 @@ +#include "ANSONNXRTMPOSE.h" +#include "Utility.h" +#include "ANSLicense.h" // ANS_DBG macro for DebugView +#include +#include +#include +#include + +namespace ANSCENTER { + + // ==================================================================== + // ONNXRtmPose — BasicOrtHandler subclass + // ==================================================================== + ONNXRtmPose::ONNXRtmPose(const std::string& _onnx_path, unsigned int _num_threads) + : BasicOrtHandler(_onnx_path, _num_threads) + { + resolveInputShape(); + } + + ONNXRtmPose::ONNXRtmPose(const std::string& _onnx_path, EngineType engineType, + unsigned int _num_threads) + : BasicOrtHandler(_onnx_path, engineType, _num_threads) + { + resolveInputShape(); + } + + // ------------------------------------------------------------------ + // resolveInputShape — read input/output dims off the model. + // Input : 1x3xHxW (RTMPose body-2d default 1x3x256x192) + // Output : simcc_x [1, K, ext_w], simcc_y [1, K, ext_h] + // The keypoint count K is read off the first output's dim[1]. + // ------------------------------------------------------------------ + void ONNXRtmPose::resolveInputShape() { + if (input_node_dims.size() >= 4) { + const int h = static_cast(input_node_dims[2]); + const int w = static_cast(input_node_dims[3]); + if (h > 0) m_input_h = h; + if (w > 0) m_input_w = w; + } + if (!output_node_dims.empty() && output_node_dims[0].size() >= 2) { + const int k = static_cast(output_node_dims[0][1]); + if (k > 0) m_num_points = k; + } + ANS_DBG("ONNXRtmPose", "resolveInputShape: %dx%d kps=%d", m_input_w, m_input_h, m_num_points); + } + + // ------------------------------------------------------------------ + // GetAffineTransform — direct port of utils.h. + // ------------------------------------------------------------------ + cv::Mat ONNXRtmPose::GetAffineTransform(float center_x, float center_y, + float scale_w, float scale_h, + int output_w, int output_h, bool inverse) + { + cv::Point2f src1(center_x, center_y); + cv::Point2f src2(center_x, center_y - scale_w * 0.5f); + cv::Point2f src3(src2.x - (src1.y - src2.y), + src2.y + (src1.x - src2.x)); + + const float dcx = output_w * 0.5f; + const float dcy = output_h * 0.5f; + cv::Point2f dst1(dcx, dcy); + cv::Point2f dst2(dcx, dcy - output_w * 0.5f); + cv::Point2f dst3(dst2.x - (dst1.y - dst2.y), + dst2.y + (dst1.x - dst2.x)); + + cv::Point2f srcPts[3] = { src1, src2, src3 }; + cv::Point2f dstPts[3] = { dst1, dst2, dst3 }; + return inverse ? cv::getAffineTransform(dstPts, srcPts) + : cv::getAffineTransform(srcPts, dstPts); + } + + // ------------------------------------------------------------------ + // CropImageByDetectBox — aspect-fit + 1.2x context expansion. + // Direct port of rtmpose_onnxruntime.cpp::CropImageByDetectBox. + // ------------------------------------------------------------------ + void ONNXRtmPose::CropImageByDetectBox(const cv::Mat& image, const cv::Rect& box, + cv::Mat& crop, cv::Mat& affine_rev) + { + float w = static_cast(box.width); + float h = static_cast(box.height); + const float cx = box.x + w * 0.5f; + const float cy = box.y + h * 0.5f; + + const float aspect = static_cast(m_input_w) / static_cast(m_input_h); + if (w > aspect * h) h = w / aspect; + else w = h * aspect; + const float scale_w = w * 1.2f; + const float scale_h = h * 1.2f; + + cv::Mat affine = GetAffineTransform(cx, cy, scale_w, scale_h, + m_input_w, m_input_h, false); + affine_rev = GetAffineTransform(cx, cy, scale_w, scale_h, + m_input_w, m_input_h, true); + + cv::warpAffine(image, crop, affine, + cv::Size(m_input_w, m_input_h), cv::INTER_LINEAR); + } + + // ------------------------------------------------------------------ + // transformInto — input is a 192x256 BGR person crop. Convert to + // RGB, ImageNet-normalize, lay out CHW into outBuffer, build the + // Ort::Value view over outBuffer.data(). Mirrors the design of + // ONNXYOLO::transformInto: per-call backing storage is owned by the + // caller so two threads sharing one ONNXRtmPose can preprocess and + // Run() concurrently without racing on input_values_handler. + // ------------------------------------------------------------------ + Ort::Value ONNXRtmPose::transformInto(const cv::Mat& cropBGR, + std::vector& outBuffer, + std::vector& outDims) + { + cv::Mat rgb; + cv::cvtColor(cropBGR, rgb, cv::COLOR_BGR2RGB); + + cv::Mat floatImg; + rgb.convertTo(floatImg, CV_32FC3); + + const int H = floatImg.rows; + const int W = floatImg.cols; + const size_t HW = static_cast(H) * W; + + outDims = { 1, 3, static_cast(H), static_cast(W) }; + outBuffer.resize(3 * HW); + float* data = outBuffer.data(); + + // HWC -> CHW + (pixel - mean) / std. RGB channel order. + for (int y = 0; y < H; ++y) { + const float* row = floatImg.ptr(y); + for (int x = 0; x < W; ++x) { + const float r = row[x * 3 + 0]; + const float g = row[x * 3 + 1]; + const float b = row[x * 3 + 2]; + data[0 * HW + y * W + x] = (r - MEAN_R) / STD_R; + data[1 * HW + y * W + x] = (g - MEAN_G) / STD_G; + data[2 * HW + y * W + x] = (b - MEAN_B) / STD_B; + } + } + + return Ort::Value::CreateTensor( + *memory_info_handler, + outBuffer.data(), + outBuffer.size(), + outDims.data(), + outDims.size()); + } + + // BasicOrtHandler pure-virtual override. Thin shim that writes into + // the shared base members (legacy single-threaded caller path). + Ort::Value ONNXRtmPose::transform(const cv::Mat& mat) + { + Ort::Value v = transformInto(mat, input_values_handler, input_node_dims); + input_tensor_size = input_values_handler.size(); + return v; + } + + // RTMPose is single-person per call — batched inference doesn't apply. + Ort::Value ONNXRtmPose::transformBatch(const std::vector& /*images*/) + { + throw std::runtime_error("ONNXRtmPose::transformBatch: not supported (single-person model)"); + } + + // ------------------------------------------------------------------ + // estimate — full per-bbox pipeline. + // ------------------------------------------------------------------ + Object ONNXRtmPose::estimate(const cv::Mat& image, const cv::Rect& box) + { + Object obj{}; + if (image.empty() || box.width <= 0 || box.height <= 0) { + ANS_DBG("ONNXRtmPose", "estimate: invalid input empty=%d box=%dx%d", + image.empty() ? 1 : 0, box.width, box.height); + return obj; + } + + try { + // Affine crop -> 192x256 BGR person image, plus inverse-affine + // to map output coords back to original image. + cv::Mat crop, affine_reverse; + CropImageByDetectBox(image, box, crop, affine_reverse); + if (crop.empty()) { + ANS_DBG("ONNXRtmPose", "estimate: warpAffine produced empty crop"); + return obj; + } + + // Per-call backing storage so multiple threads sharing one + // ONNXRtmPose can Run() in parallel — see comment on + // transformInto. + std::vector inputBuffer; + std::vector inputDims; + Ort::Value inputTensor = transformInto(crop, inputBuffer, inputDims); + + auto outputTensors = ort_session->Run( + Ort::RunOptions{ nullptr }, + input_node_names.data(), + &inputTensor, 1, + output_node_names.data(), + num_outputs); + + if (outputTensors.size() < 2) { + ANS_DBG("ONNXRtmPose", "estimate: expected 2 outputs (simcc_x,simcc_y), got %zu", + outputTensors.size()); + return obj; + } + + // Decode SimCC. Disambiguate which output is X vs Y by length + // (X output's last dim is proportional to input W; Y's to H). + float* out0 = outputTensors[0].GetTensorMutableData(); + float* out1 = outputTensors[1].GetTensorMutableData(); + auto shape0 = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape(); + auto shape1 = outputTensors[1].GetTensorTypeAndShapeInfo().GetShape(); + if (shape0.size() < 3 || shape1.size() < 3 || + shape0[1] != shape1[1] || shape0[1] <= 0) { + ANS_DBG("ONNXRtmPose", "estimate: bad output shapes (rank or K mismatch)"); + return obj; + } + const int joint_num = static_cast(shape0[1]); + int W0 = static_cast(shape0[2]); + int H1 = static_cast(shape1[2]); + float* simcc_x = out0; + float* simcc_y = out1; + int extW = W0; + int extH = H1; + // If the first output looks H-shaped (closer to 2*input_h than + // 2*input_w), swap so simcc_x always indexes the W-axis. + const float ratioFirstW = (m_input_w > 0) ? float(W0) / float(m_input_w) : 0.f; + const float ratioFirstH = (m_input_h > 0) ? float(W0) / float(m_input_h) : 0.f; + if (std::abs(ratioFirstW - 2.f) > std::abs(ratioFirstH - 2.f)) { + std::swap(simcc_x, simcc_y); + extW = H1; + extH = W0; + } + + const double m0 = affine_reverse.at(0, 0); + const double m1 = affine_reverse.at(0, 1); + const double m2 = affine_reverse.at(0, 2); + const double m3 = affine_reverse.at(1, 0); + const double m4 = affine_reverse.at(1, 1); + const double m5 = affine_reverse.at(1, 2); + + const float imgW = static_cast(image.cols); + const float imgH = static_cast(image.rows); + + std::vector kps; + kps.reserve(static_cast(joint_num) * 3); + std::vector polygon; + polygon.reserve(joint_num); + std::stringstream xss, yss; + float meanScore = 0.f; + int meanCount = 0; + + for (int i = 0; i < joint_num; ++i) { + const float* x_ptr = simcc_x + static_cast(i) * extW; + const float* y_ptr = simcc_y + static_cast(i) * extH; + const int px = static_cast(std::max_element(x_ptr, x_ptr + extW) - x_ptr); + const int py = static_cast(std::max_element(y_ptr, y_ptr + extH) - y_ptr); + const float score = std::max(x_ptr[px], y_ptr[py]); + + // SimCC oversampling factor 2 — same constant as the reference impl. + const double raw_x = px * 0.5; + const double raw_y = py * 0.5; + + float wx = static_cast(m0 * raw_x + m1 * raw_y + m2); + float wy = static_cast(m3 * raw_x + m4 * raw_y + m5); + wx = std::clamp(wx, 0.f, imgW - 1.f); + wy = std::clamp(wy, 0.f, imgH - 1.f); + + kps.push_back(wx); + kps.push_back(wy); + kps.push_back(score); + polygon.emplace_back(wx, wy); + + xss << wx; + yss << wy; + if (i + 1 < joint_num) { xss << ";"; yss << ";"; } + + if (score > 0.f) { meanScore += score; ++meanCount; } + } + + obj.classId = 0; + obj.confidence = (meanCount > 0) ? meanScore / meanCount : 0.f; + obj.box = box; + obj.kps = std::move(kps); + obj.polygon = std::move(polygon); + obj.extraInfo = xss.str() + "|" + yss.str(); + return obj; + } + catch (const std::exception& e) { + ANS_DBG("ONNXRtmPose", "estimate: EXCEPTION '%s'", e.what()); + return obj; + } + } + + // ==================================================================== + // ANSONNXRTMPOSE — ANSODBase wrapper + // ==================================================================== + ANSONNXRTMPOSE::~ANSONNXRTMPOSE() { + try { Destroy(); } + catch (const std::exception& e) { + _logger.LogError("ANSONNXRTMPOSE::~ANSONNXRTMPOSE", e.what(), __FILE__, __LINE__); + } + } + + bool ANSONNXRTMPOSE::Destroy() { + try { + m_ortEngine.reset(); + ANS_DBG("ANSONNXRTMPOSE", "Destroy: engine released"); + return true; + } + catch (const std::exception& e) { + _logger.LogError("ANSONNXRTMPOSE::Destroy", e.what(), __FILE__, __LINE__); + return false; + } + } + + bool ANSONNXRTMPOSE::OptimizeModel(bool fp16, std::string& optimizedModelFolder) { + if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) return false; + // ONNX models don't need an explicit precision-conversion pass here; + // the EP picks FP16 / FP32 at session-creation time. Mirrors + // ANSONNXYOLO::OptimizeModel which is also a no-op beyond folder + // resolution. + optimizedModelFolder = _modelFolder; + return true; + } + + bool ANSONNXRTMPOSE::InitOrtEngine() { + try { + if (!FileExist(_modelFilePath)) { + _logger.LogError("ANSONNXRTMPOSE::InitOrtEngine", + "Model file does not exist: " + _modelFilePath, __FILE__, __LINE__); + ANS_DBG("ANSONNXRTMPOSE", "InitOrtEngine: missing onnx '%s'", _modelFilePath.c_str()); + return false; + } + m_ortEngine = std::make_shared(_modelFilePath); + ANS_DBG("ANSONNXRTMPOSE", "InitOrtEngine: ready %dx%d kps=%d", + m_ortEngine->inputWidth(), m_ortEngine->inputHeight(), + m_ortEngine->numKeypoints()); + return true; + } + catch (const std::exception& e) { + _logger.LogFatal("ANSONNXRTMPOSE::InitOrtEngine", e.what(), __FILE__, __LINE__); + ANS_DBG("ANSONNXRTMPOSE", "InitOrtEngine: EXCEPTION '%s'", e.what()); + return false; + } + } + + bool ANSONNXRTMPOSE::InitOrtEngine(ANSCENTER::EngineType engineType) { + try { + if (!FileExist(_modelFilePath)) { + _logger.LogError("ANSONNXRTMPOSE::InitOrtEngine", + "Model file does not exist: " + _modelFilePath, __FILE__, __LINE__); + return false; + } + m_ortEngine = std::make_shared(_modelFilePath, engineType); + ANS_DBG("ANSONNXRTMPOSE", "InitOrtEngine(EP=%d): ready %dx%d kps=%d", + static_cast(engineType), + m_ortEngine->inputWidth(), m_ortEngine->inputHeight(), + m_ortEngine->numKeypoints()); + return true; + } + catch (const std::exception& e) { + _logger.LogFatal("ANSONNXRTMPOSE::InitOrtEngine", e.what(), __FILE__, __LINE__); + return false; + } + } + + // ------------------------------------------------------------------ + // WarmUpEngine — single dummy inference to amortise first-call cost + // (CUDA arena alloc / cuDNN algo selection / DML shader compile). + // Non-fatal on failure: the engine is fully loaded before this runs. + // ------------------------------------------------------------------ + void ANSONNXRTMPOSE::WarmUpEngine() { + if (!m_ortEngine) return; + const auto et = m_ortEngine->getEngineType(); + if (et != EngineType::NVIDIA_GPU && et != EngineType::AMD_GPU) { + ANS_DBG("ANSONNXRTMPOSE", "Warm-up skipped (CPU/OpenVINO EP)"); + return; + } + + try { + const int W = m_ortEngine->inputWidth(); + const int H = m_ortEngine->inputHeight(); + cv::Mat dummy(H, W, CV_8UC3, cv::Scalar(128, 128, 128)); + const cv::Rect wholeImage(0, 0, W, H); + + auto t0 = std::chrono::steady_clock::now(); + (void)m_ortEngine->estimate(dummy, wholeImage); + auto t1 = std::chrono::steady_clock::now(); + auto ms = std::chrono::duration_cast(t1 - t0).count(); + ANS_DBG("ANSONNXRTMPOSE", "Warm-up done: %lld ms (%dx%d)", (long long)ms, W, H); + } + catch (const std::exception& e) { + _logger.LogWarn("ANSONNXRTMPOSE::WarmUpEngine", + std::string("Warm-up skipped (non-fatal): ") + e.what(), + __FILE__, __LINE__); + ANS_DBG("ANSONNXRTMPOSE", "Warm-up: EXCEPTION '%s'", e.what()); + } + catch (...) { + _logger.LogWarn("ANSONNXRTMPOSE::WarmUpEngine", + "Warm-up skipped (unknown exception)", __FILE__, __LINE__); + } + } + + bool ANSONNXRTMPOSE::Initialize(std::string licenseKey, ModelConfig modelConfig, + const std::string& modelZipFilePath, + const std::string& modelZipPassword, + std::string& labelMap) + { + std::lock_guard lock(_mutex); + ModelLoadingGuard mlg(_modelLoading); + ANS_DBG("ANSONNXRTMPOSE", "Initialize: zip='%s'", modelZipFilePath.c_str()); + try { + _modelLoadValid = false; + bool result = ANSODBase::Initialize(licenseKey, modelConfig, + modelZipFilePath, modelZipPassword, labelMap); + if (!result) return false; + + _modelConfig = modelConfig; + _modelConfig.modelType = ModelType::ONNXRTMPOSE; + if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = 256; + if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = 192; + if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) + _modelConfig.numKPS = 26; + if (_modelConfig.kpsThreshold <= 0) + _modelConfig.kpsThreshold = KPS_THRESHOLD; + KPS_THRESHOLD = _modelConfig.kpsThreshold; + + _modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx"); + if (FileExist(_modelConfigFile)) { + ModelType modelType; + std::vector inputShape; + _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); + if (inputShape.size() == 2) { + if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; + if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; + } + } + else { + _classFilePath = CreateFilePath(_modelFolder, "classes.names"); + std::ifstream isValid(_classFilePath); + if (!isValid) LoadClassesFromString(); + else LoadClassesFromFile(); + } + + labelMap.clear(); + if (!_classes.empty()) + labelMap = VectorToCommaSeparatedString(_classes); + + if (this->_loadEngineOnCreation) { + if (!InitOrtEngine()) { + _logger.LogError("ANSONNXRTMPOSE::Initialize", + "Failed to create ONNX Runtime engine: " + _modelFilePath, + __FILE__, __LINE__); + return false; + } + // Sync ModelConfig.numKPS to whatever the model actually has, + // so consumers reading GetModelConfig() see the truth. + if (m_ortEngine && m_ortEngine->numKeypoints() > 0) + _modelConfig.numKPS = m_ortEngine->numKeypoints(); + WarmUpEngine(); + } + + _modelLoadValid = true; + _isInitialized = true; + return true; + } + catch (const std::exception& e) { + _logger.LogFatal("ANSONNXRTMPOSE::Initialize", e.what(), __FILE__, __LINE__); + ANS_DBG("ANSONNXRTMPOSE", "Initialize: EXCEPTION '%s'", e.what()); + return false; + } + } + + bool ANSONNXRTMPOSE::LoadModel(const std::string& modelZipFilePath, + const std::string& modelZipPassword) + { + std::lock_guard lock(_mutex); + ModelLoadingGuard mlg(_modelLoading); + ANS_DBG("ANSONNXRTMPOSE", "LoadModel: zip='%s'", modelZipFilePath.c_str()); + try { + bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword); + if (!result) return false; + + _modelConfig.modelType = ModelType::ONNXRTMPOSE; + if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = 256; + if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = 192; + if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) + _modelConfig.numKPS = 26; + if (_modelConfig.kpsThreshold <= 0) + _modelConfig.kpsThreshold = KPS_THRESHOLD; + KPS_THRESHOLD = _modelConfig.kpsThreshold; + + _modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx"); + if (FileExist(_modelConfigFile)) { + ModelType modelType; + std::vector inputShape; + _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); + if (inputShape.size() == 2) { + if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; + if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; + } + } + else { + _classFilePath = CreateFilePath(_modelFolder, "classes.names"); + std::ifstream isValid(_classFilePath); + if (!isValid) LoadClassesFromString(); + else LoadClassesFromFile(); + } + + if (this->_loadEngineOnCreation) { + if (!InitOrtEngine()) { _modelLoadValid = false; return false; } + if (m_ortEngine && m_ortEngine->numKeypoints() > 0) + _modelConfig.numKPS = m_ortEngine->numKeypoints(); + WarmUpEngine(); + } + _modelLoadValid = true; + _isInitialized = true; + return true; + } + catch (const std::exception& e) { + _logger.LogFatal("ANSONNXRTMPOSE::LoadModel", e.what(), __FILE__, __LINE__); + ANS_DBG("ANSONNXRTMPOSE", "LoadModel: EXCEPTION '%s'", e.what()); + return false; + } + } + + bool ANSONNXRTMPOSE::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, + std::string modelName, std::string className, + const std::string& modelFolder, + std::string& labelMap) + { + std::lock_guard lock(_mutex); + ModelLoadingGuard mlg(_modelLoading); + ANS_DBG("ANSONNXRTMPOSE", "LoadModelFromFolder: folder='%s' model='%s'", + modelFolder.c_str(), modelName.c_str()); + try { + bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, + modelName, className, + modelFolder, labelMap); + if (!result) return false; + + _modelConfig = modelConfig; + _modelConfig.modelType = ModelType::ONNXRTMPOSE; + if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = 256; + if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = 192; + if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) + _modelConfig.numKPS = 26; + if (_modelConfig.kpsThreshold <= 0) + _modelConfig.kpsThreshold = KPS_THRESHOLD; + KPS_THRESHOLD = _modelConfig.kpsThreshold; + + std::string _modelName = modelName; + if (_modelName.empty()) _modelName = "train_last"; + std::string modelFullName = _modelName + ".onnx"; + + _modelFilePath = CreateFilePath(_modelFolder, modelFullName); + if (FileExist(_modelConfigFile)) { + ModelType modelType; + std::vector inputShape; + _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); + if (inputShape.size() == 2) { + if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; + if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; + } + } + else { + _classFilePath = CreateFilePath(_modelFolder, className); + std::ifstream isValid(_classFilePath); + if (!isValid) LoadClassesFromString(); + else LoadClassesFromFile(); + } + + labelMap.clear(); + if (!_classes.empty()) + labelMap = VectorToCommaSeparatedString(_classes); + + if (this->_loadEngineOnCreation) { + if (!InitOrtEngine()) { _modelLoadValid = false; return false; } + if (m_ortEngine && m_ortEngine->numKeypoints() > 0) + _modelConfig.numKPS = m_ortEngine->numKeypoints(); + WarmUpEngine(); + } + _modelLoadValid = true; + _isInitialized = true; + return true; + } + catch (const std::exception& e) { + _logger.LogFatal("ANSONNXRTMPOSE::LoadModelFromFolder", e.what(), __FILE__, __LINE__); + ANS_DBG("ANSONNXRTMPOSE", "LoadModelFromFolder: EXCEPTION '%s'", e.what()); + return false; + } + } + + std::vector ANSONNXRTMPOSE::RunInference(const cv::Mat& inputImgBGR) { + return RunInference(inputImgBGR, ""); + } + + std::vector ANSONNXRTMPOSE::RunInference(const cv::Mat& inputImgBGR, + const std::string& camera_id) + { + if (_modelLoading.load()) return {}; + { + auto lock = TryLockWithTimeout("ANSONNXRTMPOSE::RunInference"); + if (!lock.owns_lock()) return {}; + if (!_modelLoadValid) { + _logger.LogError("ANSONNXRTMPOSE::RunInference", "Model not loaded", __FILE__, __LINE__); + return {}; + } + if (!_licenseValid) { + _logger.LogError("ANSONNXRTMPOSE::RunInference", "Invalid license", __FILE__, __LINE__); + return {}; + } + if (!_isInitialized) { + _logger.LogError("ANSONNXRTMPOSE::RunInference", "Model not initialized", __FILE__, __LINE__); + return {}; + } + if (inputImgBGR.empty() || inputImgBGR.cols < 10 || inputImgBGR.rows < 10) + return {}; + } + try { return DetectObjects(inputImgBGR, camera_id); } + catch (const std::exception& e) { + _logger.LogFatal("ANSONNXRTMPOSE::RunInference", e.what(), __FILE__, __LINE__); + ANS_DBG("ANSONNXRTMPOSE", "RunInference: EXCEPTION '%s'", e.what()); + return {}; + } + } + + // ------------------------------------------------------------------ + // DetectObjects — top-down: with no external bbox, treat the whole + // input image as the person crop and run RTMPose. Higher-level + // orchestration that already has per-person bboxes should call + // ANSODBase::RunInference(input, vector, cam) — that overload + // crops each bbox and routes back here per crop. + // ------------------------------------------------------------------ + std::vector ANSONNXRTMPOSE::DetectObjects(const cv::Mat& inputImage, + const std::string& camera_id) + { + try { + // Snapshot engine handle under brief lock; dereference outside + // (mirrors ANSONNXYOLO::DetectObjects so concurrent inference + // threads don't serialise on _mutex for the full ORT Run()). + std::shared_ptr engine; + std::string firstClass; + { + auto lk = TryLockWithTimeout("ANSONNXRTMPOSE::DetectObjects"); + if (!lk.owns_lock()) return {}; + if (!m_ortEngine) { + _logger.LogError("ANSONNXRTMPOSE::DetectObjects", + "ORT engine is null", __FILE__, __LINE__); + return {}; + } + engine = m_ortEngine; // ref-count++ + if (!_classes.empty()) firstClass = _classes.front(); + } + + const cv::Rect wholeImage(0, 0, inputImage.cols, inputImage.rows); + Object obj = engine->estimate(inputImage, wholeImage); + if (obj.kps.empty()) return {}; + + obj.cameraId = camera_id; + obj.className = firstClass.empty() ? std::string("person") : firstClass; + + std::vector out; + out.push_back(std::move(obj)); + ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: kps=%d cam='%s'", + engine->numKeypoints(), camera_id.c_str()); + return out; + } + catch (const std::exception& e) { + _logger.LogFatal("ANSONNXRTMPOSE::DetectObjects", e.what(), __FILE__, __LINE__); + ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: EXCEPTION '%s'", e.what()); + return {}; + } + } + +} // namespace ANSCENTER diff --git a/modules/ANSODEngine/ANSONNXRTMPOSE.h b/modules/ANSODEngine/ANSONNXRTMPOSE.h new file mode 100644 index 0000000..ec3f1cb --- /dev/null +++ b/modules/ANSODEngine/ANSONNXRTMPOSE.h @@ -0,0 +1,159 @@ +#ifndef ANSONNXRTMPOSE_H +#define ANSONNXRTMPOSE_H +#pragma once +#include "ANSEngineCommon.h" +#include "ONNXEngine.h" + +namespace ANSCENTER { + + // ==================================================================== + // ONNXRtmPose — BasicOrtHandler subclass for RTMPose (SimCC) ONNX + // + // Ported from C:\Projects\AIVISION\RTM_POSE\OnnxRuntime-CPU\ + // OnnxRunTime-CPU\rtmpose_onnxruntime.cpp but using BasicOrtHandler + // so the session benefits from the shared EP loader (CUDA / TensorRT + // / DirectML / OpenVINO / CPU auto-detect), per-session high-perf + // options, EP fallback recovery, and the wstring path handling that + // BasicOrtHandler centralises. + // + // RTMPose is a top-down single-person pose estimator: + // Input : person crop, 192x256 RGB, ImageNet-normalized + // Output : two SimCC heatmaps + // simcc_x [1, K, ext_w] + // simcc_y [1, K, ext_h] + // K = number of keypoints (e.g. 17 / 26 / 133) + // ext_w / ext_h = SimCC oversampled dims (typ. 2 * input) + // + // Class name is mixed-case (not ONNXRTMPOSE) to avoid a collision with + // the unscoped enum value ModelType::ONNXRTMPOSE in ANSLicense.h — + // identical spelling would shadow the type. + // ==================================================================== + class ONNXENGINE_API ONNXRtmPose : public BasicOrtHandler + { + public: + explicit ONNXRtmPose(const std::string& _onnx_path, + unsigned int _num_threads = 1); + explicit ONNXRtmPose(const std::string& _onnx_path, + EngineType engineType, + unsigned int _num_threads = 1); + ~ONNXRtmPose() override = default; + + // Full per-bbox pipeline. Returns one Object with kps in original- + // image coords (kps layout: [x, y, score] * K). Empty kps means + // the call short-circuited (invalid box, runtime error, etc.). + Object estimate(const cv::Mat& image, const cv::Rect& box); + + // Read-back accessors so the wrapper can introspect once the + // session is alive (used to populate _modelConfig.numKPS, log a + // sanity line, etc.). + int inputWidth() const { return m_input_w; } + int inputHeight() const { return m_input_h; } + int numKeypoints() const { return m_num_points; } + + private: + // Override the BasicOrtHandler pure virtuals. transform() takes a + // pre-cropped 192x256 BGR person image and produces the normalized + // CHW tensor. transformBatch() throws — RTMPose is single-person. + Ort::Value transform(const cv::Mat& mat) override; + Ort::Value transformBatch(const std::vector& images) override; + + // Per-call preprocess that writes into a caller-provided buffer + + // dims vector instead of the BasicOrtHandler members (mirrors + // ONNXYOLO::transformInto). Lets concurrent threads sharing one + // ONNXRtmPose instance preprocess and infer in parallel without + // racing on input_values_handler. + Ort::Value transformInto(const cv::Mat& cropBGR, + std::vector& outBuffer, + std::vector& outDims); + + // Affine helpers — direct port of utils.h::GetAffineTransform + // and rtmpose_onnxruntime.cpp::CropImageByDetectBox. + static cv::Mat GetAffineTransform(float center_x, float center_y, + float scale_w, float scale_h, + int output_w, int output_h, + bool inverse = false); + void CropImageByDetectBox(const cv::Mat& image, const cv::Rect& box, + cv::Mat& crop, cv::Mat& affine_rev); + + // Read input geometry off the model (1x3xHxW). Falls back to + // 192x256 (the canonical RTMPose body-2d shape) if the ONNX + // export uses dynamic dims. + void resolveInputShape(); + + // Fixed RTMPose model geometry (read off the model at construction; + // defaults match the reference impl). + int m_input_w{ 192 }; + int m_input_h{ 256 }; + int m_num_points{ 26 }; + + // ImageNet normalization. Matches the IMAGE_MEAN / IMAGE_STD + // constants in C:\Projects\AIVISION\RTM_POSE\OnnxRuntime-CPU\ + // OnnxRunTime-CPU\rtmpose_utils.h. + const float MEAN_R{ 123.675f }; + const float MEAN_G{ 116.28f }; + const float MEAN_B{ 103.53f }; + const float STD_R { 58.395f }; + const float STD_G { 57.12f }; + const float STD_B { 57.375f }; + }; + + // ==================================================================== + // ANSONNXRTMPOSE — ANSODBase wrapper around ONNXRtmPose + // + // Mirrors the structure of ANSONNXYOLO. Owns a shared_ptr + // so a concurrent inference thread can copy the handle under a brief + // lock and dereference it lock-free, which lets multiple AI tasks + // share one ANSONNXRTMPOSE without serialising on the engine mutex. + // ==================================================================== + class ANSENGINE_API ANSONNXRTMPOSE : public ANSODBase { + public: + bool Initialize(std::string licenseKey, ModelConfig modelConfig, + const std::string& modelZipFilePath, + const std::string& modelZipPassword, + std::string& labelMap) override; + + bool LoadModel(const std::string& modelZipFilePath, + const std::string& modelZipPassword) override; + + bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, + std::string modelName, std::string className, + const std::string& modelFolder, + std::string& labelMap) override; + + bool OptimizeModel(bool fp16, std::string& optimizedModelFolder) override; + + std::vector RunInference(const cv::Mat& input) override; + std::vector RunInference(const cv::Mat& input, + const std::string& camera_id) override; + + bool Destroy() override; + ~ANSONNXRTMPOSE(); + + private: + std::string _modelFilePath; + bool _modelLoadValid{ false }; + + // Keypoint confidence floor. The SimCC max score is emitted as + // kps[2] regardless; this is just the threshold used to compute + // the per-Object confidence average. + float KPS_THRESHOLD{ 0.3f }; + + // ORT engine. shared_ptr (mirrors ANSONNXYOLO) so an inference + // thread holding a local copy keeps the underlying session alive + // even if a future CPU-fallback / EP-rebuild path resets the + // member here on another thread. + std::shared_ptr m_ortEngine; + + bool InitOrtEngine(); + bool InitOrtEngine(ANSCENTER::EngineType engineType); + void WarmUpEngine(); + + // Per-bbox pipeline used by RunInference. Pulled out as its own + // method so the future bbox-overload (ANSODBase::RunInference with + // a vector) can route here directly. + std::vector DetectObjects(const cv::Mat& inputImage, + const std::string& camera_id); + }; + +} // namespace ANSCENTER +#endif diff --git a/modules/ANSODEngine/ANSRTRTMPOSE.cpp b/modules/ANSODEngine/ANSRTRTMPOSE.cpp new file mode 100644 index 0000000..bee658e --- /dev/null +++ b/modules/ANSODEngine/ANSRTRTMPOSE.cpp @@ -0,0 +1,702 @@ +#include "ANSRTRTMPOSE.h" +#include "Utility.h" +#include "ANSLicense.h" // ANS_DBG macro for DebugView +#include +#include +#include +#include + +namespace ANSCENTER { + + // ==================================================================== + // ANSODBase interface — OptimizeModel + // ==================================================================== + bool ANSRTRTMPOSE::OptimizeModel(bool fp16, std::string& optimizedModelFolder) { + std::lock_guard lock(_mutex); + ANS_DBG("ANSRTRTMPOSE", "OptimizeModel: fp16=%d", fp16 ? 1 : 0); + if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) return false; + if (!FileExist(_modelFilePath)) { + _logger.LogFatal("ANSRTRTMPOSE::OptimizeModel", + "Raw model file path does not exist: " + _modelFilePath, __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "OptimizeModel: missing onnx '%s'", _modelFilePath.c_str()); + return false; + } + try { + _fp16 = fp16; + optimizedModelFolder = GetParentFolder(_modelFilePath); + if (!m_trtEngine) { + m_options.optBatchSize = _modelConfig.gpuOptBatchSize; + m_options.maxBatchSize = _modelConfig.gpuMaxBatchSize; + m_options.deviceIndex = _modelConfig.gpuDeviceIndex; + m_options.maxInputHeight = _modelConfig.maxInputHeight; + m_options.minInputHeight = _modelConfig.minInputHeight; + m_options.optInputHeight = _modelConfig.optInputHeight; + m_options.maxInputWidth = _modelConfig.maxInputWidth; + m_options.minInputWidth = _modelConfig.minInputWidth; + m_options.optInputWidth = _modelConfig.optInputWidth; + m_options.engineFileDir = optimizedModelFolder; + m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32); + m_trtEngine = std::make_shared>(m_options); + } + auto succ = m_trtEngine->buildWithRetry(_modelFilePath, SUB_VALS, DIV_VALS, NORMALIZE); + if (!succ) { + _logger.LogError("ANSRTRTMPOSE::OptimizeModel", + "Error: Unable to build TensorRT engine. " + _modelFilePath, __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "OptimizeModel: buildWithRetry FAILED for '%s'", _modelFilePath.c_str()); + return false; + } + ANS_DBG("ANSRTRTMPOSE", "OptimizeModel: build OK -> '%s'", optimizedModelFolder.c_str()); + return true; + } + catch (std::exception& e) { + _logger.LogFatal("ANSRTRTMPOSE::OptimizeModel", e.what(), __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "OptimizeModel: EXCEPTION '%s'", e.what()); + return false; + } + } + + // ==================================================================== + // ANSODBase interface — LoadModel + // ==================================================================== + bool ANSRTRTMPOSE::LoadModel(const std::string& modelZipFilePath, + const std::string& modelZipPassword) { + std::lock_guard lock(_mutex); + ModelLoadingGuard mlg(_modelLoading); + ANS_DBG("ANSRTRTMPOSE", "LoadModel: zip='%s'", modelZipFilePath.c_str()); + try { + bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword); + if (!result) return false; + + _modelConfig.modelType = ModelType::RTRTMPOSE; + // RTMPose has fixed input dims — we still propagate them through ModelConfig + // so the rest of the pipeline (caching, logging) can introspect them. + if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = RTM_INPUT_H; + if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = RTM_INPUT_W; + if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) + _modelConfig.numKPS = 26; // default to RTMPose body 2d-26 + if (_modelConfig.kpsThreshold <= 0) + _modelConfig.kpsThreshold = KPS_THRESHOLD; + _fp16 = true; + KPS_THRESHOLD = _modelConfig.kpsThreshold; + + m_options.optBatchSize = _modelConfig.gpuOptBatchSize; + m_options.maxBatchSize = _modelConfig.gpuMaxBatchSize; + m_options.deviceIndex = _modelConfig.gpuDeviceIndex; + m_options.maxInputHeight = _modelConfig.maxInputHeight; + m_options.minInputHeight = _modelConfig.minInputHeight; + m_options.optInputHeight = _modelConfig.optInputHeight; + m_options.maxInputWidth = _modelConfig.maxInputWidth; + m_options.minInputWidth = _modelConfig.minInputWidth; + m_options.optInputWidth = _modelConfig.optInputWidth; + m_options.engineFileDir = _modelFolder; + m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32); + + _modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx"); + if (FileExist(_modelConfigFile)) { + ModelType modelType; + std::vector inputShape; + _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); + if (inputShape.size() == 2) { + if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; + if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; + } + } + else { + _classFilePath = CreateFilePath(_modelFolder, "classes.names"); + std::ifstream isValid(_classFilePath); + if (!isValid) LoadClassesFromString(); + else LoadClassesFromFile(); + } + + if (this->_loadEngineOnCreation) { + if (!m_trtEngine) { + m_poolKey = { _modelFilePath, + static_cast(m_options.precision), m_options.maxBatchSize }; + m_trtEngine = EnginePoolManager::instance().acquire( + m_poolKey, m_options, _modelFilePath, + SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu); + m_usingSharedPool = (m_trtEngine != nullptr); + } + if (!m_trtEngine) { + _logger.LogError("ANSRTRTMPOSE::LoadModel", + "Error: Unable to load TensorRT engine. " + _modelFilePath, __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "LoadModel: engine acquire FAILED '%s'", _modelFilePath.c_str()); + _modelLoadValid = false; + return false; + } + m_options.maxBatchSize = m_trtEngine->getOptions().maxBatchSize; + m_options.optBatchSize = m_trtEngine->getOptions().optBatchSize; + m_trtEngine->warmUp(); + + // Pull true input geometry from the engine; RTMPose is fixed-size, + // but reading it back protects us against unexpected ONNX exports. + const auto& inputDims = m_trtEngine->getInputDims(); + if (!inputDims.empty()) { + if (inputDims[0].d[1] > 0) RTM_INPUT_H = inputDims[0].d[1]; + if (inputDims[0].d[2] > 0) RTM_INPUT_W = inputDims[0].d[2]; + } + ANS_DBG("ANSRTRTMPOSE", "LoadModel: engine ready %dx%d kps=%d sharedPool=%d", + RTM_INPUT_W, RTM_INPUT_H, _modelConfig.numKPS, m_usingSharedPool ? 1 : 0); + } + _modelLoadValid = true; + _isInitialized = true; + return true; + } + catch (std::exception& e) { + _logger.LogFatal("ANSRTRTMPOSE::LoadModel", e.what(), __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "LoadModel: EXCEPTION '%s'", e.what()); + return false; + } + } + + // ==================================================================== + // ANSODBase interface — LoadModelFromFolder + // ==================================================================== + bool ANSRTRTMPOSE::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, + std::string modelName, std::string className, + const std::string& modelFolder, std::string& labelMap) { + std::lock_guard lock(_mutex); + ModelLoadingGuard mlg(_modelLoading); + ANS_DBG("ANSRTRTMPOSE", "LoadModelFromFolder: folder='%s' model='%s'", + modelFolder.c_str(), modelName.c_str()); + try { + bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, + modelName, className, modelFolder, labelMap); + if (!result) return false; + + _modelConfig = modelConfig; + _modelConfig.modelType = ModelType::RTRTMPOSE; + if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = RTM_INPUT_H; + if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = RTM_INPUT_W; + _modelConfig.precisionType = PrecisionType::FP32; + if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) + _modelConfig.numKPS = 26; + if (_modelConfig.kpsThreshold <= 0) + _modelConfig.kpsThreshold = KPS_THRESHOLD; + _fp16 = true; + KPS_THRESHOLD = _modelConfig.kpsThreshold; + + std::string _modelName = modelName; + if (_modelName.empty()) _modelName = "train_last"; + std::string modelFullName = _modelName + ".onnx"; + + m_options.optBatchSize = _modelConfig.gpuOptBatchSize; + m_options.maxBatchSize = _modelConfig.gpuMaxBatchSize; + m_options.deviceIndex = _modelConfig.gpuDeviceIndex; + m_options.maxInputHeight = _modelConfig.maxInputHeight; + m_options.minInputHeight = _modelConfig.minInputHeight; + m_options.optInputHeight = _modelConfig.optInputHeight; + m_options.maxInputWidth = _modelConfig.maxInputWidth; + m_options.minInputWidth = _modelConfig.minInputWidth; + m_options.optInputWidth = _modelConfig.optInputWidth; + m_options.engineFileDir = _modelFolder; + m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32); + + _modelFilePath = CreateFilePath(_modelFolder, modelFullName); + if (FileExist(_modelConfigFile)) { + ModelType modelType; + std::vector inputShape; + _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); + if (inputShape.size() == 2) { + if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; + if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; + } + } + else { + _classFilePath = CreateFilePath(_modelFolder, className); + std::ifstream isValid(_classFilePath); + if (!isValid) LoadClassesFromString(); + else LoadClassesFromFile(); + } + + labelMap.clear(); + if (!_classes.empty()) + labelMap = VectorToCommaSeparatedString(_classes); + + if (this->_loadEngineOnCreation) { + if (!m_trtEngine) { + m_poolKey = { _modelFilePath, + static_cast(m_options.precision), m_options.maxBatchSize }; + m_trtEngine = EnginePoolManager::instance().acquire( + m_poolKey, m_options, _modelFilePath, + SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu); + m_usingSharedPool = (m_trtEngine != nullptr); + } + if (!m_trtEngine) { + _logger.LogError("ANSRTRTMPOSE::LoadModelFromFolder", + "Error: Unable to load TensorRT engine. " + _modelFilePath, __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "LoadModelFromFolder: engine acquire FAILED '%s'", + _modelFilePath.c_str()); + _modelLoadValid = false; + return false; + } + m_options.maxBatchSize = m_trtEngine->getOptions().maxBatchSize; + m_options.optBatchSize = m_trtEngine->getOptions().optBatchSize; + m_trtEngine->warmUp(); + + const auto& inputDims = m_trtEngine->getInputDims(); + if (!inputDims.empty()) { + if (inputDims[0].d[1] > 0) RTM_INPUT_H = inputDims[0].d[1]; + if (inputDims[0].d[2] > 0) RTM_INPUT_W = inputDims[0].d[2]; + } + ANS_DBG("ANSRTRTMPOSE", "LoadModelFromFolder: engine ready %dx%d kps=%d sharedPool=%d", + RTM_INPUT_W, RTM_INPUT_H, _modelConfig.numKPS, m_usingSharedPool ? 1 : 0); + } + _modelLoadValid = true; + _isInitialized = true; + return true; + } + catch (std::exception& e) { + _logger.LogFatal("ANSRTRTMPOSE::LoadModelFromFolder", e.what(), __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "LoadModelFromFolder: EXCEPTION '%s'", e.what()); + return false; + } + } + + // ==================================================================== + // ANSODBase interface — Initialize + // ==================================================================== + bool ANSRTRTMPOSE::Initialize(std::string licenseKey, ModelConfig modelConfig, + const std::string& modelZipFilePath, + const std::string& modelZipPassword, + std::string& labelMap) { + std::lock_guard lock(_mutex); + ModelLoadingGuard mlg(_modelLoading); + ANS_DBG("ANSRTRTMPOSE", "Initialize: zip='%s'", modelZipFilePath.c_str()); + try { + const bool engineAlreadyLoaded = + _modelLoadValid && _isInitialized && m_trtEngine != nullptr; + _modelLoadValid = false; + bool result = ANSODBase::Initialize(licenseKey, modelConfig, + modelZipFilePath, modelZipPassword, labelMap); + if (!result) return false; + + _modelConfig = modelConfig; + _modelConfig.modelType = ModelType::RTRTMPOSE; + if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = RTM_INPUT_H; + if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = RTM_INPUT_W; + _modelConfig.precisionType = PrecisionType::FP32; + if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) + _modelConfig.numKPS = 26; + if (_modelConfig.kpsThreshold <= 0) + _modelConfig.kpsThreshold = KPS_THRESHOLD; + _fp16 = true; + KPS_THRESHOLD = _modelConfig.kpsThreshold; + + m_options.optBatchSize = _modelConfig.gpuOptBatchSize; + m_options.maxBatchSize = _modelConfig.gpuMaxBatchSize; + m_options.deviceIndex = _modelConfig.gpuDeviceIndex; + m_options.maxInputHeight = _modelConfig.maxInputHeight; + m_options.minInputHeight = _modelConfig.minInputHeight; + m_options.optInputHeight = _modelConfig.optInputHeight; + m_options.maxInputWidth = _modelConfig.maxInputWidth; + m_options.minInputWidth = _modelConfig.minInputWidth; + m_options.optInputWidth = _modelConfig.optInputWidth; + m_options.engineFileDir = _modelFolder; + m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32); + + _modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx"); + if (FileExist(_modelConfigFile)) { + ModelType modelType; + std::vector inputShape; + _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); + if (inputShape.size() == 2) { + if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; + if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; + } + } + else { + _classFilePath = CreateFilePath(_modelFolder, "classes.names"); + std::ifstream isValid(_classFilePath); + if (!isValid) LoadClassesFromString(); + else LoadClassesFromFile(); + } + + labelMap.clear(); + if (!_classes.empty()) + labelMap = VectorToCommaSeparatedString(_classes); + + if (this->_loadEngineOnCreation && !engineAlreadyLoaded) { + if (!m_trtEngine) { + m_poolKey = { _modelFilePath, + static_cast(m_options.precision), m_options.maxBatchSize }; + m_trtEngine = EnginePoolManager::instance().acquire( + m_poolKey, m_options, _modelFilePath, + SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu); + m_usingSharedPool = (m_trtEngine != nullptr); + } + if (!m_trtEngine) { + _logger.LogError("ANSRTRTMPOSE::Initialize", + "Error: Unable to load TensorRT engine. " + _modelFilePath, __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "Initialize: engine acquire FAILED '%s'", _modelFilePath.c_str()); + _modelLoadValid = false; + return false; + } + m_options.maxBatchSize = m_trtEngine->getOptions().maxBatchSize; + m_options.optBatchSize = m_trtEngine->getOptions().optBatchSize; + m_trtEngine->warmUp(); + + const auto& inputDims = m_trtEngine->getInputDims(); + if (!inputDims.empty()) { + if (inputDims[0].d[1] > 0) RTM_INPUT_H = inputDims[0].d[1]; + if (inputDims[0].d[2] > 0) RTM_INPUT_W = inputDims[0].d[2]; + } + ANS_DBG("ANSRTRTMPOSE", "Initialize: engine ready %dx%d kps=%d sharedPool=%d", + RTM_INPUT_W, RTM_INPUT_H, _modelConfig.numKPS, m_usingSharedPool ? 1 : 0); + } + _modelLoadValid = true; + _isInitialized = true; + return true; + } + catch (std::exception& e) { + _logger.LogFatal("ANSRTRTMPOSE::Initialize", e.what(), __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "Initialize: EXCEPTION '%s'", e.what()); + return false; + } + } + + // ==================================================================== + // RunInference / Destroy / Destructor + // ==================================================================== + std::vector ANSRTRTMPOSE::RunInference(const cv::Mat& inputImgBGR) { + return RunInference(inputImgBGR, ""); + } + + std::vector ANSRTRTMPOSE::RunInference(const cv::Mat& inputImgBGR, + const std::string& camera_id) { + if (!PreInferenceCheck("ANSRTRTMPOSE::RunInference")) return {}; + if (inputImgBGR.empty()) { + ANS_DBG("ANSRTRTMPOSE", "RunInference: empty input cam='%s'", camera_id.c_str()); + return {}; + } + try { + // Top-down pose estimator: with no external bbox, treat the whole + // image as a single person crop. Higher-level orchestration that + // wants per-person pose should call ANSODBase::RunInference(input, + // bboxes, cam) instead — that overload crops each bbox and routes + // back through us per crop. + const cv::Rect wholeImage(0, 0, inputImgBGR.cols, inputImgBGR.rows); + return InferenceOnBox(inputImgBGR, wholeImage, camera_id); + } + catch (const std::exception& e) { + _logger.LogFatal("ANSRTRTMPOSE::RunInference", e.what(), __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "RunInference: EXCEPTION '%s'", e.what()); + return {}; + } + } + + ANSRTRTMPOSE::~ANSRTRTMPOSE() { + try { Destroy(); } + catch (std::exception& e) { + _logger.LogError("ANSRTRTMPOSE::~ANSRTRTMPOSE()", e.what(), __FILE__, __LINE__); + } + } + + bool ANSRTRTMPOSE::Destroy() { + try { + if (m_usingSharedPool) { + EnginePoolManager::instance().release(m_poolKey); + m_trtEngine.reset(); + m_usingSharedPool = false; + } + else { + m_trtEngine.reset(); + } + ANS_DBG("ANSRTRTMPOSE", "Destroy: engine released"); + return true; + } + catch (std::exception& e) { + _logger.LogError("ANSRTRTMPOSE::Destroy()", e.what(), __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "Destroy: EXCEPTION '%s'", e.what()); + return false; + } + } + + // ==================================================================== + // Per-bbox single-shot inference pipeline + // ==================================================================== + std::vector ANSRTRTMPOSE::InferenceOnBox(const cv::Mat& input, + const cv::Rect& box, + const std::string& camera_id) { + if (!m_trtEngine) { + ANS_DBG("ANSRTRTMPOSE", "InferenceOnBox: engine null"); + return {}; + } + if (box.width <= 0 || box.height <= 0) { + ANS_DBG("ANSRTRTMPOSE", "InferenceOnBox: invalid box %dx%d", box.width, box.height); + return {}; + } + + m_trtEngine->setDeviceContext(); + + // Phase 1 — preprocess under lock (affine warp on CPU, upload to GPU) + ImageMetadata meta; + std::vector> trtInput; + { + std::lock_guard lock(_mutex); + trtInput = Preprocess(input, box, meta); + } + if (trtInput.empty() || trtInput[0].empty()) { + ANS_DBG("ANSRTRTMPOSE", "InferenceOnBox: preprocess failed box=%d,%d %dx%d", + box.x, box.y, box.width, box.height); + return {}; + } + + // Phase 2 — inference (mutex released; pool dispatches to idle slot) + std::vector>> featureVectors; + if (!m_trtEngine->runInference(trtInput, featureVectors)) { + _logger.LogError("ANSRTRTMPOSE::InferenceOnBox", "runInference failed", __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "InferenceOnBox: runInference FAILED"); + return {}; + } + if (featureVectors.empty() || featureVectors[0].size() < 2) { + _logger.LogError("ANSRTRTMPOSE::InferenceOnBox", + "Expected 2 SimCC outputs (x,y), got " + + std::to_string(featureVectors.empty() ? 0 : featureVectors[0].size()), + __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "InferenceOnBox: bad output count outer=%zu inner=%zu", + featureVectors.size(), + featureVectors.empty() ? size_t{ 0 } : featureVectors[0].size()); + return {}; + } + + // Phase 3 — postprocess. Output ordering in featureVectors matches the + // engine's output binding order; for RTMPose ONNX exports that's + // [simcc_x, simcc_y]. We disambiguate by feature length anyway. + const auto& outputDims = m_trtEngine->getOutputDims(); + if (outputDims.size() < 2 || + outputDims[0].nbDims < 3 || outputDims[1].nbDims < 3) { + _logger.LogError("ANSRTRTMPOSE::InferenceOnBox", + "Unexpected output dims (need 2 outputs of rank>=3)", __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "InferenceOnBox: outputDims rank issue count=%zu", + outputDims.size()); + return {}; + } + const int K0 = outputDims[0].d[1]; + const int W0 = outputDims[0].d[2]; + const int K1 = outputDims[1].d[1]; + const int H1 = outputDims[1].d[2]; + if (K0 != K1) { + _logger.LogError("ANSRTRTMPOSE::InferenceOnBox", + "SimCC keypoint count mismatch x=" + std::to_string(K0) + + " y=" + std::to_string(K1), __FILE__, __LINE__); + return {}; + } + // Pick which output is X (longer of the model-input dims) vs Y. + // For 192x256 input, ext_w=384 is shorter than ext_h=512, but the order + // of the two SimCC outputs in the ONNX is fixed, so we identify by + // length proportional to RTM_INPUT_W vs RTM_INPUT_H rather than by + // assuming x-first. + const std::vector* simcc_x = &featureVectors[0][0]; + const std::vector* simcc_y = &featureVectors[0][1]; + int extW = W0; + int extH = H1; + const float ratioFirst = (RTM_INPUT_W > 0) + ? static_cast(W0) / static_cast(RTM_INPUT_W) : 0.f; + const float ratioSecond = (RTM_INPUT_H > 0) + ? static_cast(W0) / static_cast(RTM_INPUT_H) : 0.f; + // If the first output's last-dim looks like H (proportional to input H, + // not W), swap so simcc_x indexes the W-axis. + if (std::abs(ratioFirst - 2.f) > std::abs(ratioSecond - 2.f)) { + std::swap(simcc_x, simcc_y); + extW = H1; + extH = W0; + } + + std::lock_guard lock(_mutex); + Object obj = PostprocessSimCC(*simcc_x, *simcc_y, K0, extW, extH, meta, camera_id); + std::vector out; + if (!obj.kps.empty()) out.push_back(std::move(obj)); + ANS_DBG("ANSRTRTMPOSE", "InferenceOnBox: kps=%d cam='%s'", + out.empty() ? 0 : K0, camera_id.c_str()); + return out; + } + + // ==================================================================== + // Preprocess — affine crop -> upload BGR (Engine handles RGB norm) + // + // NOTE on color order: Engine::runInference calls blobFromGpuMats + // with swapRB=false (see EngineRunInference.inl). So the GpuMat we hand + // it is interpreted channel-by-channel as the model input. RTMPose was + // trained on RGB, so we cvtColor BGR -> RGB before upload (mirroring + // ANSTENSORRTPOSE / ANSRTYOLO). + // ==================================================================== + std::vector> ANSRTRTMPOSE::Preprocess( + const cv::Mat& inputImage, const cv::Rect& box, ImageMetadata& outMeta) { + try { + if (!_licenseValid) { + _logger.LogFatal("ANSRTRTMPOSE::Preprocess", "Invalid license", __FILE__, __LINE__); + return {}; + } + + cv::Mat srcImg = inputImage; + if (srcImg.channels() == 1) { + cv::cvtColor(srcImg, srcImg, cv::COLOR_GRAY2BGR); + } + outMeta.imgHeight = static_cast(srcImg.rows); + outMeta.imgWidth = static_cast(srcImg.cols); + outMeta.personBox = box; + if (outMeta.imgHeight <= 0 || outMeta.imgWidth <= 0) return {}; + + cv::Mat crop; + CropImageByDetectBox(srcImg, box, crop, outMeta.affine_rev); + if (crop.empty()) { + ANS_DBG("ANSRTRTMPOSE", "Preprocess: warpAffine produced empty crop"); + return {}; + } + + cv::Mat rgb; + cv::cvtColor(crop, rgb, cv::COLOR_BGR2RGB); + + cv::cuda::Stream stream; + cv::cuda::GpuMat gpuRGB; + gpuRGB.upload(rgb, stream); + stream.waitForCompletion(); + + std::vector input{ std::move(gpuRGB) }; + std::vector> inputs{ std::move(input) }; + return inputs; + } + catch (const std::exception& e) { + _logger.LogWarn("ANSRTRTMPOSE::Preprocess", + std::string("Skipped frame: ") + e.what(), __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "Preprocess: EXCEPTION '%s'", e.what()); + return {}; + } + } + + // ==================================================================== + // PostprocessSimCC + // + // For each keypoint k: + // px = argmax(simcc_x[k]) ; sx = simcc_x[k][px] + // py = argmax(simcc_y[k]) ; sy = simcc_y[k][py] + // score = max(sx, sy) + // raw_x = px * 0.5 ; raw_y = py * 0.5 + // [x, y]^T = affine_rev * [raw_x, raw_y, 1]^T + // + // This matches RTMPose::Inference in rtmpose_tensorrt.cpp. + // ==================================================================== + Object ANSRTRTMPOSE::PostprocessSimCC(const std::vector& simcc_x, + const std::vector& simcc_y, + int numKeypoints, int extW, int extH, + const ImageMetadata& meta, + const std::string& camera_id) { + Object obj{}; + if (numKeypoints <= 0 || extW <= 0 || extH <= 0) return obj; + if (static_cast(simcc_x.size()) < numKeypoints * extW || + static_cast(simcc_y.size()) < numKeypoints * extH) { + _logger.LogError("ANSRTRTMPOSE::PostprocessSimCC", + "SimCC buffer underflow: x=" + std::to_string(simcc_x.size()) + + " y=" + std::to_string(simcc_y.size()) + + " expected " + std::to_string(numKeypoints * extW) + "/" + + std::to_string(numKeypoints * extH), __FILE__, __LINE__); + return obj; + } + + const double m0 = meta.affine_rev.at(0, 0); + const double m1 = meta.affine_rev.at(0, 1); + const double m2 = meta.affine_rev.at(0, 2); + const double m3 = meta.affine_rev.at(1, 0); + const double m4 = meta.affine_rev.at(1, 1); + const double m5 = meta.affine_rev.at(1, 2); + + std::vector kps; + kps.reserve(numKeypoints * 3); + std::vector polygon; + polygon.reserve(numKeypoints); + std::stringstream xss, yss; + float meanScore = 0.f; + int meanCount = 0; + + for (int i = 0; i < numKeypoints; ++i) { + const float* xPtr = simcc_x.data() + static_cast(i) * extW; + const float* yPtr = simcc_y.data() + static_cast(i) * extH; + const int px = static_cast(std::max_element(xPtr, xPtr + extW) - xPtr); + const int py = static_cast(std::max_element(yPtr, yPtr + extH) - yPtr); + const float score = std::max(xPtr[px], yPtr[py]); + + // SimCC oversampling factor 2 — same constant as the reference impl. + const double raw_x = px * 0.5; + const double raw_y = py * 0.5; + + float worldX = static_cast(m0 * raw_x + m1 * raw_y + m2); + float worldY = static_cast(m3 * raw_x + m4 * raw_y + m5); + + // Clamp into image bounds so downstream code can index safely. + worldX = std::clamp(worldX, 0.f, meta.imgWidth - 1.f); + worldY = std::clamp(worldY, 0.f, meta.imgHeight - 1.f); + + kps.push_back(worldX); + kps.push_back(worldY); + kps.push_back(score); + polygon.emplace_back(worldX, worldY); + + xss << worldX; + yss << worldY; + if (i + 1 < numKeypoints) { xss << ";"; yss << ";"; } + + if (score >= KPS_THRESHOLD) { meanScore += score; ++meanCount; } + } + + obj.classId = 0; + obj.className = _classes.empty() ? std::string("person") : _classes.front(); + obj.confidence = (meanCount > 0) ? meanScore / meanCount : 0.f; + obj.box = meta.personBox; + obj.kps = std::move(kps); + obj.polygon = std::move(polygon); + obj.cameraId = camera_id; + obj.extraInfo = xss.str() + "|" + yss.str(); + return obj; + } + + // ==================================================================== + // Affine helpers — direct port of utils.h / rtmpose_tensorrt.cpp + // ==================================================================== + cv::Mat ANSRTRTMPOSE::GetAffineTransform(float center_x, float center_y, + float scale_w, float scale_h, + int output_w, int output_h, bool inverse) { + cv::Point2f src1(center_x, center_y); + cv::Point2f src2(center_x, center_y - scale_w * 0.5f); + cv::Point2f src3(src2.x - (src1.y - src2.y), + src2.y + (src1.x - src2.x)); + + const float dcx = output_w * 0.5f; + const float dcy = output_h * 0.5f; + cv::Point2f dst1(dcx, dcy); + cv::Point2f dst2(dcx, dcy - output_w * 0.5f); + cv::Point2f dst3(dst2.x - (dst1.y - dst2.y), + dst2.y + (dst1.x - dst2.x)); + + cv::Point2f srcPts[3] = { src1, src2, src3 }; + cv::Point2f dstPts[3] = { dst1, dst2, dst3 }; + return inverse ? cv::getAffineTransform(dstPts, srcPts) + : cv::getAffineTransform(srcPts, dstPts); + } + + void ANSRTRTMPOSE::CropImageByDetectBox(const cv::Mat& image, const cv::Rect& box, + cv::Mat& crop, cv::Mat& affine_rev) { + // Aspect-fit the model's W:H (e.g. 192:256) over the bbox, then expand + // by 1.2x for context — same heuristic as the reference RTMPose impl. + float w = static_cast(box.width); + float h = static_cast(box.height); + const float cx = box.x + w * 0.5f; + const float cy = box.y + h * 0.5f; + + const float aspect = static_cast(RTM_INPUT_W) / + static_cast(RTM_INPUT_H); + if (w > aspect * h) h = w / aspect; + else w = h * aspect; + const float scale_w = w * 1.2f; + const float scale_h = h * 1.2f; + + cv::Mat affine = GetAffineTransform(cx, cy, scale_w, scale_h, + RTM_INPUT_W, RTM_INPUT_H, false); + affine_rev = GetAffineTransform(cx, cy, scale_w, scale_h, + RTM_INPUT_W, RTM_INPUT_H, true); + + cv::warpAffine(image, crop, affine, + cv::Size(RTM_INPUT_W, RTM_INPUT_H), cv::INTER_LINEAR); + } + +} // namespace ANSCENTER diff --git a/modules/ANSODEngine/ANSRTRTMPOSE.h b/modules/ANSODEngine/ANSRTRTMPOSE.h new file mode 100644 index 0000000..f706d1f --- /dev/null +++ b/modules/ANSODEngine/ANSRTRTMPOSE.h @@ -0,0 +1,125 @@ +#ifndef ANSRTRTMPOSE_H +#define ANSRTRTMPOSE_H +#pragma once +#include "ANSEngineCommon.h" +#include "engine.h" +#include "ANSGpuFrameRegistry.h" +#include "engine/EnginePoolManager.h" + +namespace ANSCENTER { + + // ==================================================================== + // ANSRTRTMPOSE — TensorRT-based RTMPose (SimCC) pose estimator + // + // Ported from C:\Projects\AIVISION\RTM_POSE\TensorRT\rtmpose_tensorrt.cpp + // but uses Engine from TensorRTAPI (shared pool / pinned upload / + // multi-GPU dispatch / engine cache) instead of raw nvinfer1. + // + // RTMPose is a top-down single-person pose estimator: + // Input : person crop, 192x256 RGB, ImageNet-normalized + // Output : two SimCC heatmaps + // simcc_x [1, K, ext_w] + // simcc_y [1, K, ext_h] + // K = number of keypoints (e.g. 17 / 26 / 133) + // ext_w / ext_h = SimCC oversampled dims (typ. 2 * input) + // + // RunInference(image) treats the whole input as one person crop. To run + // on multiple people, call ANSODBase::RunInference(image, bboxes, cam) — + // the base class crops each bbox and dispatches to RunInference per crop. + // ==================================================================== + class ANSENGINE_API ANSRTRTMPOSE : public ANSODBase { + public: + bool Initialize(std::string licenseKey, ModelConfig modelConfig, + const std::string& modelZipFilePath, + const std::string& modelZipPassword, + std::string& labelMap) override; + + bool LoadModel(const std::string& modelZipFilePath, + const std::string& modelZipPassword) override; + + bool LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, + std::string modelName, std::string className, + const std::string& modelFolder, + std::string& labelMap) override; + + bool OptimizeModel(bool fp16, std::string& optimizedModelFolder) override; + + std::vector RunInference(const cv::Mat& input); + std::vector RunInference(const cv::Mat& input, + const std::string& camera_id); + + bool Destroy(); + ~ANSRTRTMPOSE(); + + private: + std::string _modelFilePath; + bool _modelLoadValid{ false }; + bool _fp16{ false }; + int m_maxSlotsPerGpu{ 1 }; + void SetMaxSlotsPerGpu(int n) override { m_maxSlotsPerGpu = n; } + + // Per-call image geometry — captured before inference, consumed at postprocess. + // affine_rev maps SimCC output coords (in 192x256 model space) back to + // the original image. Carrying this per-call keeps the class re-entrant. + struct ImageMetadata { + cv::Mat affine_rev; + float imgWidth = 0.f; + float imgHeight = 0.f; + cv::Rect personBox; // bbox the affine was built from + }; + + // Single-bbox pipeline: affine-crop -> upload -> Engine.runInference -> SimCC decode + std::vector InferenceOnBox(const cv::Mat& input, + const cv::Rect& box, + const std::string& camera_id); + + // Build the 192x256 RGB crop around `box` and upload to GPU. outMeta + // captures the inverse-affine and original image size for postprocess. + std::vector> Preprocess(const cv::Mat& inputImage, + const cv::Rect& box, + ImageMetadata& outMeta); + + // Decode SimCC outputs into one Object with kps in original-image coords. + Object PostprocessSimCC(const std::vector& simcc_x, + const std::vector& simcc_y, + int numKeypoints, int extW, int extH, + const ImageMetadata& meta, + const std::string& camera_id); + + // Affine helpers — direct port of utils.h::GetAffineTransform and + // rtmpose_tensorrt.cpp::CropImageByDetectBox. + static cv::Mat GetAffineTransform(float center_x, float center_y, + float scale_w, float scale_h, + int output_w, int output_h, + bool inverse = false); + void CropImageByDetectBox(const cv::Mat& image, const cv::Rect& box, + cv::Mat& crop, cv::Mat& affine_rev); + + // ── TensorRT engine (shared pool, same pattern as ANSRTYOLO) ───── + std::shared_ptr> m_trtEngine = nullptr; + EnginePoolManager::PoolKey m_poolKey; + bool m_usingSharedPool = false; + + ANSCENTER::Options m_options; + + // Fixed RTMPose model geometry (overridden from the engine's input dims + // at LoadModel time once the network is built). + int RTM_INPUT_W{ 192 }; + int RTM_INPUT_H{ 256 }; + + // ImageNet normalization, expressed for Engine::blobFromGpuMats: + // blob = (pixel/255 - subVals) / divVals + // subVals = mean / 255 ; divVals = std / 255 + const std::array SUB_VALS{ 0.485f, 0.456f, 0.406f }; + const std::array DIV_VALS{ 0.229f, 0.224f, 0.225f }; + const bool NORMALIZE = true; + + // Keypoint confidence floor for output. Mirrors KPS_THRESHOLD from + // ANSTENSORRTPOSE — keypoints below threshold are still emitted (the + // SimCC max is taken regardless), but caller can read kps[2] (score) + // to decide whether to draw them. + float KPS_THRESHOLD{ 0.3f }; + }; + +} // namespace ANSCENTER +#endif diff --git a/modules/ANSODEngine/CMakeLists.txt b/modules/ANSODEngine/CMakeLists.txt index 2fbf4dd..1b12522 100644 --- a/modules/ANSODEngine/CMakeLists.txt +++ b/modules/ANSODEngine/CMakeLists.txt @@ -18,6 +18,7 @@ set(ANSOD_SOURCES ANSONNXCL.cpp ANSONNXOBB.cpp ANSONNXPOSE.cpp + ANSONNXRTMPOSE.cpp ANSONNXSAM3.cpp ANSONNXSEG.cpp ANSONNXYOLO.cpp @@ -27,6 +28,7 @@ set(ANSOD_SOURCES ANSOVSEG.cpp ANSPOSE.cpp ANSRTYOLO.cpp + ANSRTRTMPOSE.cpp ANSSAM.cpp ANSSAM3.cpp ANSTENSORRTCL.cpp diff --git a/modules/ANSODEngine/dllmain.cpp b/modules/ANSODEngine/dllmain.cpp index 88fcd2d..b42ba9d 100644 --- a/modules/ANSODEngine/dllmain.cpp +++ b/modules/ANSODEngine/dllmain.cpp @@ -52,6 +52,8 @@ extern "C" ANSODENGINE_API GpuFrameData** ANSODEngine_GetTlsGpuFrame() { #include "ANSONNXSAM3.h" #include "ANSRTYOLO.h" #include "ANSONNXYOLO.h" +#include "ANSRTRTMPOSE.h" +#include "ANSONNXRTMPOSE.h" #include #include #include "utils/visualizer.hpp" @@ -445,6 +447,11 @@ extern "C" ANSODENGINE_API std::string CreateANSODHandle(ANSCENTER::ANSODBase** case 22: // RTPOSE modelType = onNvidia ? 22 /* RTPOSE */ : 21 /* ONNXPOSE */; break; + // ── RTMPose (top-down SimCC pose) ──────────────────────────────────── + case 32: // ONNXRTMPOSE + case 33: // RTRTMPOSE + modelType = onNvidia ? 33 /* RTRTMPOSE */ : 32 /* ONNXRTMPOSE */; + break; // ── Segmentation ───────────────────────────────────────────────────── case 23: // ONNXSEG case 24: // RTSEG @@ -699,6 +706,20 @@ extern "C" ANSODENGINE_API std::string CreateANSODHandle(ANSCENTER::ANSODBase** modelConfig.modelType = ANSCENTER::ModelType::ONNXYOLO; } break; + case 32: //ONNXRTMPOSE + (*Handle) = new ANSCENTER::ANSONNXRTMPOSE(); + modelConfig.modelType = ANSCENTER::ModelType::ONNXRTMPOSE; + break; + case 33: //RTRTMPOSE + if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) { + (*Handle) = new ANSCENTER::ANSRTRTMPOSE(); + modelConfig.modelType = ANSCENTER::ModelType::RTRTMPOSE; + } + else { + (*Handle) = new ANSCENTER::ANSONNXRTMPOSE(); + modelConfig.modelType = ANSCENTER::ModelType::ONNXRTMPOSE; + } + break; default: (*Handle) = new ANSCENTER::ANSFD(); modelConfig.modelType = ANSCENTER::ModelType::FACEDETECT; @@ -803,6 +824,11 @@ extern "C" ANSODENGINE_API int CreateANSODHandleEx(ANSCENTER::ANSODBase** Handl case 22: // RTPOSE modelType = onNvidia ? 22 /* RTPOSE */ : 21 /* ONNXPOSE */; break; + // ── RTMPose (top-down SimCC pose) ──────────────────────────────────── + case 32: // ONNXRTMPOSE + case 33: // RTRTMPOSE + modelType = onNvidia ? 33 /* RTRTMPOSE */ : 32 /* ONNXRTMPOSE */; + break; // ── Segmentation ───────────────────────────────────────────────────── case 23: // ONNXSEG case 24: // RTSEG @@ -1066,6 +1092,20 @@ extern "C" ANSODENGINE_API int CreateANSODHandleEx(ANSCENTER::ANSODBase** Handl modelConfig.modelType = ANSCENTER::ModelType::ONNXYOLO; } break; + case 32: //ONNXRTMPOSE + (*Handle) = new ANSCENTER::ANSONNXRTMPOSE(); + modelConfig.modelType = ANSCENTER::ModelType::ONNXRTMPOSE; + break; + case 33: //RTRTMPOSE + if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) { + (*Handle) = new ANSCENTER::ANSRTRTMPOSE(); + modelConfig.modelType = ANSCENTER::ModelType::RTRTMPOSE; + } + else { + (*Handle) = new ANSCENTER::ANSONNXRTMPOSE(); + modelConfig.modelType = ANSCENTER::ModelType::ONNXRTMPOSE; + } + break; default: (*Handle) = new ANSCENTER::ANSFD(); modelConfig.modelType = ANSCENTER::ModelType::FACEDETECT; @@ -1207,6 +1247,11 @@ extern "C" __declspec(dllexport) int LoadModelFromFolder(ANSCENTER::ANSODBase** case 22: // RTPOSE modelType = onNvidia ? 22 /* RTPOSE */ : 21 /* ONNXPOSE */; break; + // ── RTMPose (top-down SimCC pose) ──────────────────────────────── + case 32: // ONNXRTMPOSE + case 33: // RTRTMPOSE + modelType = onNvidia ? 33 /* RTRTMPOSE */ : 32 /* ONNXRTMPOSE */; + break; // ── Segmentation ───────────────────────────────────────────────── case 23: // ONNXSEG case 24: // RTSEG @@ -1459,6 +1504,20 @@ extern "C" __declspec(dllexport) int LoadModelFromFolder(ANSCENTER::ANSODBase** modelConfig.modelType = ANSCENTER::ModelType::ONNXYOLO; } break; + case 32: //ONNXRTMPOSE + (*Handle) = new ANSCENTER::ANSONNXRTMPOSE(); + modelConfig.modelType = ANSCENTER::ModelType::ONNXRTMPOSE; + break; + case 33: //RTRTMPOSE + if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) { + (*Handle) = new ANSCENTER::ANSRTRTMPOSE(); + modelConfig.modelType = ANSCENTER::ModelType::RTRTMPOSE; + } + else { + (*Handle) = new ANSCENTER::ANSONNXRTMPOSE(); + modelConfig.modelType = ANSCENTER::ModelType::ONNXRTMPOSE; + } + break; default: (*Handle) = new ANSCENTER::ANSFD(); modelConfig.modelType = ANSCENTER::ModelType::FACEDETECT; @@ -1530,6 +1589,11 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode case 22: // RTPOSE modelType = onNvidia ? 22 /* RTPOSE */ : 21 /* ONNXPOSE */; break; + // ── RTMPose (top-down SimCC pose) ──────────────────────────────── + case 32: // ONNXRTMPOSE + case 33: // RTRTMPOSE + modelType = onNvidia ? 33 /* RTRTMPOSE */ : 32 /* ONNXRTMPOSE */; + break; // ── Segmentation ───────────────────────────────────────────────── case 23: // ONNXSEG case 24: // RTSEG @@ -1545,7 +1609,7 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode } } - + if (modelType == 31) // If modelType is RTYOLO (31), handle separately. { if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) { @@ -1596,6 +1660,16 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode } break; } + case 32: Handle = std::make_unique(); modelConfig.modelType = ANSCENTER::ModelType::ONNXRTMPOSE; break; + case 33: { + if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) { + Handle = std::make_unique(); modelConfig.modelType = ANSCENTER::ModelType::RTRTMPOSE; + } + else { + Handle = std::make_unique(); modelConfig.modelType = ANSCENTER::ModelType::ONNXRTMPOSE; + } + break; + } default: { if (modelDetectionType == 0) // classification