#include "ANSONNXRTMPOSE.h" #include "Utility.h" #include "ANSLicense.h" // ANS_DBG macro for DebugView #include #include #include #include namespace { // ── IoU dedup for end-to-end YOLO person detector output ────────────── // // Ultralytics E2E ONNX exports bake NMS into the graph, but its IoU // threshold is set at export time (default ≈ 0.7) and we can't override // it from outside. For a top-down pose pipeline that's too lax — pairs // of overlapping bboxes survive and we end up running RTMPose twice on // the same person. // // This filter walks the detector's output (already sorted by confidence // in the E2E format), keeping each box only if its IoU with every // already-kept box is below kPersonDedupIoU. O(N²) is fine because N is // capped at 300 and typically <10 in practice. constexpr float kPersonDedupIoU = 0.45f; inline float boxIoU(const cv::Rect& a, const cv::Rect& b) { const cv::Rect inter = a & b; if (inter.area() <= 0) return 0.f; const int uni = a.area() + b.area() - inter.area(); return uni > 0 ? static_cast(inter.area()) / static_cast(uni) : 0.f; } std::vector dedupByIoU( const std::vector& persons, float iouThr) { std::vector kept; kept.reserve(persons.size()); for (const auto& p : persons) { bool isDup = false; for (const auto& k : kept) { if (boxIoU(p.box, k.box) > iouThr) { isDup = true; break; } } if (!isDup) kept.push_back(p); } return kept; } } // anonymous namespace namespace ANSCENTER { // ==================================================================== // ONNXRtmPose — BasicOrtHandler subclass // ==================================================================== ONNXRtmPose::ONNXRtmPose(const std::string& _onnx_path, unsigned int _num_threads) : BasicOrtHandler(_onnx_path, _num_threads) { resolveInputShape(); } ONNXRtmPose::ONNXRtmPose(const std::string& _onnx_path, EngineType engineType, unsigned int _num_threads) : BasicOrtHandler(_onnx_path, engineType, _num_threads) { resolveInputShape(); } // ------------------------------------------------------------------ // resolveInputShape — read input/output dims off the model. // Input : 1x3xHxW (RTMPose body-2d default 1x3x256x192) // Output : simcc_x [1, K, ext_w], simcc_y [1, K, ext_h] // The keypoint count K is read off the first output's dim[1]. // ------------------------------------------------------------------ void ONNXRtmPose::resolveInputShape() { if (input_node_dims.size() >= 4) { const int h = static_cast(input_node_dims[2]); const int w = static_cast(input_node_dims[3]); if (h > 0) m_input_h = h; if (w > 0) m_input_w = w; } if (!output_node_dims.empty() && output_node_dims[0].size() >= 2) { const int k = static_cast(output_node_dims[0][1]); if (k > 0) m_num_points = k; } ANS_DBG("ONNXRtmPose", "resolveInputShape: %dx%d kps=%d", m_input_w, m_input_h, m_num_points); } // ------------------------------------------------------------------ // GetAffineTransform — direct port of utils.h. // ------------------------------------------------------------------ cv::Mat ONNXRtmPose::GetAffineTransform(float center_x, float center_y, float scale_w, float scale_h, int output_w, int output_h, bool inverse) { cv::Point2f src1(center_x, center_y); cv::Point2f src2(center_x, center_y - scale_w * 0.5f); cv::Point2f src3(src2.x - (src1.y - src2.y), src2.y + (src1.x - src2.x)); const float dcx = output_w * 0.5f; const float dcy = output_h * 0.5f; cv::Point2f dst1(dcx, dcy); cv::Point2f dst2(dcx, dcy - output_w * 0.5f); cv::Point2f dst3(dst2.x - (dst1.y - dst2.y), dst2.y + (dst1.x - dst2.x)); cv::Point2f srcPts[3] = { src1, src2, src3 }; cv::Point2f dstPts[3] = { dst1, dst2, dst3 }; return inverse ? cv::getAffineTransform(dstPts, srcPts) : cv::getAffineTransform(srcPts, dstPts); } // ------------------------------------------------------------------ // CropImageByDetectBox — aspect-fit + 1.2x context expansion. // Direct port of rtmpose_onnxruntime.cpp::CropImageByDetectBox. // ------------------------------------------------------------------ void ONNXRtmPose::CropImageByDetectBox(const cv::Mat& image, const cv::Rect& box, cv::Mat& crop, cv::Mat& affine_rev) { float w = static_cast(box.width); float h = static_cast(box.height); const float cx = box.x + w * 0.5f; const float cy = box.y + h * 0.5f; const float aspect = static_cast(m_input_w) / static_cast(m_input_h); if (w > aspect * h) h = w / aspect; else w = h * aspect; const float scale_w = w * 1.2f; const float scale_h = h * 1.2f; cv::Mat affine = GetAffineTransform(cx, cy, scale_w, scale_h, m_input_w, m_input_h, false); affine_rev = GetAffineTransform(cx, cy, scale_w, scale_h, m_input_w, m_input_h, true); cv::warpAffine(image, crop, affine, cv::Size(m_input_w, m_input_h), cv::INTER_LINEAR); } // ------------------------------------------------------------------ // transformInto — input is a 192x256 BGR person crop. Convert to // RGB, ImageNet-normalize, lay out CHW into outBuffer, build the // Ort::Value view over outBuffer.data(). Mirrors the design of // ONNXYOLO::transformInto: per-call backing storage is owned by the // caller so two threads sharing one ONNXRtmPose can preprocess and // Run() concurrently without racing on input_values_handler. // ------------------------------------------------------------------ Ort::Value ONNXRtmPose::transformInto(const cv::Mat& cropBGR, std::vector& outBuffer, std::vector& outDims) { cv::Mat rgb; cv::cvtColor(cropBGR, rgb, cv::COLOR_BGR2RGB); cv::Mat floatImg; rgb.convertTo(floatImg, CV_32FC3); const int H = floatImg.rows; const int W = floatImg.cols; const size_t HW = static_cast(H) * W; outDims = { 1, 3, static_cast(H), static_cast(W) }; outBuffer.resize(3 * HW); float* data = outBuffer.data(); // HWC -> CHW + (pixel - mean) / std. RGB channel order. for (int y = 0; y < H; ++y) { const float* row = floatImg.ptr(y); for (int x = 0; x < W; ++x) { const float r = row[x * 3 + 0]; const float g = row[x * 3 + 1]; const float b = row[x * 3 + 2]; data[0 * HW + y * W + x] = (r - MEAN_R) / STD_R; data[1 * HW + y * W + x] = (g - MEAN_G) / STD_G; data[2 * HW + y * W + x] = (b - MEAN_B) / STD_B; } } return Ort::Value::CreateTensor( *memory_info_handler, outBuffer.data(), outBuffer.size(), outDims.data(), outDims.size()); } // BasicOrtHandler pure-virtual override. Thin shim that writes into // the shared base members (legacy single-threaded caller path). Ort::Value ONNXRtmPose::transform(const cv::Mat& mat) { Ort::Value v = transformInto(mat, input_values_handler, input_node_dims); input_tensor_size = input_values_handler.size(); return v; } // RTMPose is single-person per call — batched inference doesn't apply. Ort::Value ONNXRtmPose::transformBatch(const std::vector& /*images*/) { throw std::runtime_error("ONNXRtmPose::transformBatch: not supported (single-person model)"); } // ------------------------------------------------------------------ // estimate — full per-bbox pipeline. // ------------------------------------------------------------------ Object ONNXRtmPose::estimate(const cv::Mat& image, const cv::Rect& box) { Object obj{}; if (image.empty() || box.width <= 0 || box.height <= 0) { ANS_DBG("ONNXRtmPose", "estimate: invalid input empty=%d box=%dx%d", image.empty() ? 1 : 0, box.width, box.height); return obj; } try { // Affine crop -> 192x256 BGR person image, plus inverse-affine // to map output coords back to original image. cv::Mat crop, affine_reverse; CropImageByDetectBox(image, box, crop, affine_reverse); if (crop.empty()) { ANS_DBG("ONNXRtmPose", "estimate: warpAffine produced empty crop"); return obj; } // Per-call backing storage so multiple threads sharing one // ONNXRtmPose can Run() in parallel — see comment on // transformInto. std::vector inputBuffer; std::vector inputDims; Ort::Value inputTensor = transformInto(crop, inputBuffer, inputDims); auto outputTensors = ort_session->Run( Ort::RunOptions{ nullptr }, input_node_names.data(), &inputTensor, 1, output_node_names.data(), num_outputs); if (outputTensors.size() < 2) { ANS_DBG("ONNXRtmPose", "estimate: expected 2 outputs (simcc_x,simcc_y), got %zu", outputTensors.size()); return obj; } // Decode SimCC. Disambiguate which output is X vs Y by length // (X output's last dim is proportional to input W; Y's to H). float* out0 = outputTensors[0].GetTensorMutableData(); float* out1 = outputTensors[1].GetTensorMutableData(); auto shape0 = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape(); auto shape1 = outputTensors[1].GetTensorTypeAndShapeInfo().GetShape(); if (shape0.size() < 3 || shape1.size() < 3 || shape0[1] != shape1[1] || shape0[1] <= 0) { ANS_DBG("ONNXRtmPose", "estimate: bad output shapes (rank or K mismatch)"); return obj; } const int joint_num = static_cast(shape0[1]); int W0 = static_cast(shape0[2]); int H1 = static_cast(shape1[2]); float* simcc_x = out0; float* simcc_y = out1; int extW = W0; int extH = H1; // If the first output looks H-shaped (closer to 2*input_h than // 2*input_w), swap so simcc_x always indexes the W-axis. const float ratioFirstW = (m_input_w > 0) ? float(W0) / float(m_input_w) : 0.f; const float ratioFirstH = (m_input_h > 0) ? float(W0) / float(m_input_h) : 0.f; if (std::abs(ratioFirstW - 2.f) > std::abs(ratioFirstH - 2.f)) { std::swap(simcc_x, simcc_y); extW = H1; extH = W0; } const double m0 = affine_reverse.at(0, 0); const double m1 = affine_reverse.at(0, 1); const double m2 = affine_reverse.at(0, 2); const double m3 = affine_reverse.at(1, 0); const double m4 = affine_reverse.at(1, 1); const double m5 = affine_reverse.at(1, 2); const float imgW = static_cast(image.cols); const float imgH = static_cast(image.rows); std::vector kps; kps.reserve(static_cast(joint_num) * 3); std::vector polygon; polygon.reserve(joint_num); std::stringstream xss, yss; float meanScore = 0.f; int meanCount = 0; for (int i = 0; i < joint_num; ++i) { const float* x_ptr = simcc_x + static_cast(i) * extW; const float* y_ptr = simcc_y + static_cast(i) * extH; const int px = static_cast(std::max_element(x_ptr, x_ptr + extW) - x_ptr); const int py = static_cast(std::max_element(y_ptr, y_ptr + extH) - y_ptr); const float score = std::max(x_ptr[px], y_ptr[py]); // SimCC oversampling factor 2 — same constant as the reference impl. const double raw_x = px * 0.5; const double raw_y = py * 0.5; float wx = static_cast(m0 * raw_x + m1 * raw_y + m2); float wy = static_cast(m3 * raw_x + m4 * raw_y + m5); wx = std::clamp(wx, 0.f, imgW - 1.f); wy = std::clamp(wy, 0.f, imgH - 1.f); kps.push_back(wx); kps.push_back(wy); kps.push_back(score); polygon.emplace_back(wx, wy); xss << wx; yss << wy; if (i + 1 < joint_num) { xss << ";"; yss << ";"; } if (score > 0.f) { meanScore += score; ++meanCount; } } obj.classId = 0; obj.confidence = (meanCount > 0) ? meanScore / meanCount : 0.f; obj.box = box; obj.kps = std::move(kps); obj.polygon = std::move(polygon); obj.extraInfo = xss.str() + "|" + yss.str(); return obj; } catch (const std::exception& e) { ANS_DBG("ONNXRtmPose", "estimate: EXCEPTION '%s'", e.what()); return obj; } } // ==================================================================== // ANSONNXRTMPOSE — ANSODBase wrapper // ==================================================================== ANSONNXRTMPOSE::~ANSONNXRTMPOSE() { try { Destroy(); } catch (const std::exception& e) { _logger.LogError("ANSONNXRTMPOSE::~ANSONNXRTMPOSE", e.what(), __FILE__, __LINE__); } } bool ANSONNXRTMPOSE::Destroy() { try { if (m_personDetector) { m_personDetector->Destroy(); m_personDetector.reset(); } m_ortEngine.reset(); ANS_DBG("ANSONNXRTMPOSE", "Destroy: engine released"); return true; } catch (const std::exception& e) { _logger.LogError("ANSONNXRTMPOSE::Destroy", e.what(), __FILE__, __LINE__); return false; } } bool ANSONNXRTMPOSE::OptimizeModel(bool fp16, std::string& optimizedModelFolder) { if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) return false; // ONNX models don't need an explicit precision-conversion pass here; // the EP picks FP16 / FP32 at session-creation time. Mirrors // ANSONNXYOLO::OptimizeModel which is also a no-op beyond folder // resolution. optimizedModelFolder = _modelFolder; // Also bring up the bundled person-detector if present so its ORT // session is constructed (and any cached engine is materialized) // alongside the pose model. Mirrors the TRT counterpart's behavior. (void)TryLoadPersonDetector(_modelFolder); return true; } bool ANSONNXRTMPOSE::InitOrtEngine() { try { if (!FileExist(_modelFilePath)) { _logger.LogError("ANSONNXRTMPOSE::InitOrtEngine", "Model file does not exist: " + _modelFilePath, __FILE__, __LINE__); ANS_DBG("ANSONNXRTMPOSE", "InitOrtEngine: missing onnx '%s'", _modelFilePath.c_str()); return false; } m_ortEngine = std::make_shared(_modelFilePath); ANS_DBG("ANSONNXRTMPOSE", "InitOrtEngine: ready %dx%d kps=%d", m_ortEngine->inputWidth(), m_ortEngine->inputHeight(), m_ortEngine->numKeypoints()); return true; } catch (const std::exception& e) { _logger.LogFatal("ANSONNXRTMPOSE::InitOrtEngine", e.what(), __FILE__, __LINE__); ANS_DBG("ANSONNXRTMPOSE", "InitOrtEngine: EXCEPTION '%s'", e.what()); return false; } } bool ANSONNXRTMPOSE::InitOrtEngine(ANSCENTER::EngineType engineType) { try { if (!FileExist(_modelFilePath)) { _logger.LogError("ANSONNXRTMPOSE::InitOrtEngine", "Model file does not exist: " + _modelFilePath, __FILE__, __LINE__); return false; } m_ortEngine = std::make_shared(_modelFilePath, engineType); ANS_DBG("ANSONNXRTMPOSE", "InitOrtEngine(EP=%d): ready %dx%d kps=%d", static_cast(engineType), m_ortEngine->inputWidth(), m_ortEngine->inputHeight(), m_ortEngine->numKeypoints()); return true; } catch (const std::exception& e) { _logger.LogFatal("ANSONNXRTMPOSE::InitOrtEngine", e.what(), __FILE__, __LINE__); return false; } } // ------------------------------------------------------------------ // WarmUpEngine — single dummy inference to amortise first-call cost // (CUDA arena alloc / cuDNN algo selection / DML shader compile). // Non-fatal on failure: the engine is fully loaded before this runs. // ------------------------------------------------------------------ void ANSONNXRTMPOSE::WarmUpEngine() { if (!m_ortEngine) return; const auto et = m_ortEngine->getEngineType(); if (et != EngineType::NVIDIA_GPU && et != EngineType::AMD_GPU) { ANS_DBG("ANSONNXRTMPOSE", "Warm-up skipped (CPU/OpenVINO EP)"); return; } try { const int W = m_ortEngine->inputWidth(); const int H = m_ortEngine->inputHeight(); cv::Mat dummy(H, W, CV_8UC3, cv::Scalar(128, 128, 128)); const cv::Rect wholeImage(0, 0, W, H); auto t0 = std::chrono::steady_clock::now(); (void)m_ortEngine->estimate(dummy, wholeImage); auto t1 = std::chrono::steady_clock::now(); auto ms = std::chrono::duration_cast(t1 - t0).count(); ANS_DBG("ANSONNXRTMPOSE", "Warm-up done: %lld ms (%dx%d)", (long long)ms, W, H); } catch (const std::exception& e) { _logger.LogWarn("ANSONNXRTMPOSE::WarmUpEngine", std::string("Warm-up skipped (non-fatal): ") + e.what(), __FILE__, __LINE__); ANS_DBG("ANSONNXRTMPOSE", "Warm-up: EXCEPTION '%s'", e.what()); } catch (...) { _logger.LogWarn("ANSONNXRTMPOSE::WarmUpEngine", "Warm-up skipped (unknown exception)", __FILE__, __LINE__); } } bool ANSONNXRTMPOSE::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) { std::lock_guard lock(_mutex); ModelLoadingGuard mlg(_modelLoading); ANS_DBG("ANSONNXRTMPOSE", "Initialize: zip='%s'", modelZipFilePath.c_str()); try { _modelLoadValid = false; bool result = ANSODBase::Initialize(licenseKey, modelConfig, modelZipFilePath, modelZipPassword, labelMap); if (!result) return false; _modelConfig = modelConfig; _modelConfig.modelType = ModelType::ONNXRTMPOSE; if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = 256; if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = 192; if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) _modelConfig.numKPS = 26; if (_modelConfig.kpsThreshold <= 0) _modelConfig.kpsThreshold = KPS_THRESHOLD; KPS_THRESHOLD = _modelConfig.kpsThreshold; _modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx"); if (FileExist(_modelConfigFile)) { ModelType modelType; std::vector inputShape; _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); if (inputShape.size() == 2) { if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; } } else { _classFilePath = CreateFilePath(_modelFolder, "classes.names"); std::ifstream isValid(_classFilePath); if (!isValid) LoadClassesFromString(); else LoadClassesFromFile(); } labelMap.clear(); if (!_classes.empty()) labelMap = VectorToCommaSeparatedString(_classes); if (this->_loadEngineOnCreation) { if (!InitOrtEngine()) { _logger.LogError("ANSONNXRTMPOSE::Initialize", "Failed to create ONNX Runtime engine: " + _modelFilePath, __FILE__, __LINE__); return false; } // Sync ModelConfig.numKPS to whatever the model actually has, // so consumers reading GetModelConfig() see the truth. if (m_ortEngine && m_ortEngine->numKeypoints() > 0) _modelConfig.numKPS = m_ortEngine->numKeypoints(); WarmUpEngine(); } // Top-down pipeline: try to load the bundled person detector. // Falls back gracefully (whole-image pose) when the files are missing. (void)TryLoadPersonDetector(_modelFolder); _modelLoadValid = true; _isInitialized = true; return true; } catch (const std::exception& e) { _logger.LogFatal("ANSONNXRTMPOSE::Initialize", e.what(), __FILE__, __LINE__); ANS_DBG("ANSONNXRTMPOSE", "Initialize: EXCEPTION '%s'", e.what()); return false; } } bool ANSONNXRTMPOSE::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) { std::lock_guard lock(_mutex); ModelLoadingGuard mlg(_modelLoading); ANS_DBG("ANSONNXRTMPOSE", "LoadModel: zip='%s'", modelZipFilePath.c_str()); try { bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword); if (!result) return false; _modelConfig.modelType = ModelType::ONNXRTMPOSE; if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = 256; if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = 192; if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) _modelConfig.numKPS = 26; if (_modelConfig.kpsThreshold <= 0) _modelConfig.kpsThreshold = KPS_THRESHOLD; KPS_THRESHOLD = _modelConfig.kpsThreshold; _modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx"); if (FileExist(_modelConfigFile)) { ModelType modelType; std::vector inputShape; _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); if (inputShape.size() == 2) { if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; } } else { _classFilePath = CreateFilePath(_modelFolder, "classes.names"); std::ifstream isValid(_classFilePath); if (!isValid) LoadClassesFromString(); else LoadClassesFromFile(); } if (this->_loadEngineOnCreation) { if (!InitOrtEngine()) { _modelLoadValid = false; return false; } if (m_ortEngine && m_ortEngine->numKeypoints() > 0) _modelConfig.numKPS = m_ortEngine->numKeypoints(); WarmUpEngine(); } (void)TryLoadPersonDetector(_modelFolder); _modelLoadValid = true; _isInitialized = true; return true; } catch (const std::exception& e) { _logger.LogFatal("ANSONNXRTMPOSE::LoadModel", e.what(), __FILE__, __LINE__); ANS_DBG("ANSONNXRTMPOSE", "LoadModel: EXCEPTION '%s'", e.what()); return false; } } bool ANSONNXRTMPOSE::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, std::string className, const std::string& modelFolder, std::string& labelMap) { std::lock_guard lock(_mutex); ModelLoadingGuard mlg(_modelLoading); ANS_DBG("ANSONNXRTMPOSE", "LoadModelFromFolder: folder='%s' model='%s'", modelFolder.c_str(), modelName.c_str()); try { bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig, modelName, className, modelFolder, labelMap); if (!result) return false; _modelConfig = modelConfig; _modelConfig.modelType = ModelType::ONNXRTMPOSE; if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = 256; if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = 192; if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133) _modelConfig.numKPS = 26; if (_modelConfig.kpsThreshold <= 0) _modelConfig.kpsThreshold = KPS_THRESHOLD; KPS_THRESHOLD = _modelConfig.kpsThreshold; std::string _modelName = modelName; if (_modelName.empty()) _modelName = "train_last"; std::string modelFullName = _modelName + ".onnx"; _modelFilePath = CreateFilePath(_modelFolder, modelFullName); if (FileExist(_modelConfigFile)) { ModelType modelType; std::vector inputShape; _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); if (inputShape.size() == 2) { if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; } } else { _classFilePath = CreateFilePath(_modelFolder, className); std::ifstream isValid(_classFilePath); if (!isValid) LoadClassesFromString(); else LoadClassesFromFile(); } labelMap.clear(); if (!_classes.empty()) labelMap = VectorToCommaSeparatedString(_classes); if (this->_loadEngineOnCreation) { if (!InitOrtEngine()) { _modelLoadValid = false; return false; } if (m_ortEngine && m_ortEngine->numKeypoints() > 0) _modelConfig.numKPS = m_ortEngine->numKeypoints(); WarmUpEngine(); } (void)TryLoadPersonDetector(_modelFolder); _modelLoadValid = true; _isInitialized = true; return true; } catch (const std::exception& e) { _logger.LogFatal("ANSONNXRTMPOSE::LoadModelFromFolder", e.what(), __FILE__, __LINE__); ANS_DBG("ANSONNXRTMPOSE", "LoadModelFromFolder: EXCEPTION '%s'", e.what()); return false; } } std::vector ANSONNXRTMPOSE::RunInference(const cv::Mat& inputImgBGR) { return RunInference(inputImgBGR, ""); } std::vector ANSONNXRTMPOSE::RunInference(const cv::Mat& inputImgBGR, const std::string& camera_id) { if (_modelLoading.load()) return {}; { auto lock = TryLockWithTimeout("ANSONNXRTMPOSE::RunInference"); if (!lock.owns_lock()) return {}; if (!_modelLoadValid) { _logger.LogError("ANSONNXRTMPOSE::RunInference", "Model not loaded", __FILE__, __LINE__); return {}; } if (!_licenseValid) { _logger.LogError("ANSONNXRTMPOSE::RunInference", "Invalid license", __FILE__, __LINE__); return {}; } if (!_isInitialized) { _logger.LogError("ANSONNXRTMPOSE::RunInference", "Model not initialized", __FILE__, __LINE__); return {}; } if (inputImgBGR.empty() || inputImgBGR.cols < 10 || inputImgBGR.rows < 10) return {}; } try { return DetectObjects(inputImgBGR, camera_id); } catch (const std::exception& e) { _logger.LogFatal("ANSONNXRTMPOSE::RunInference", e.what(), __FILE__, __LINE__); ANS_DBG("ANSONNXRTMPOSE", "RunInference: EXCEPTION '%s'", e.what()); return {}; } } // ------------------------------------------------------------------ // DetectObjects — two pipelines: // // (a) Top-down (preferred) — m_personDetector loaded from the // bundled person.onnx + person.names. Detect people first, // then run RTMPose ONCE per person bbox. This is what RTMPose // is designed for; works on multi-person scenes with small / // distant subjects. // // (b) Standalone fallback — no detector files in the model folder. // Treat the whole image as a single person crop. Only sensible // for inputs that are already a person (selfies, single-subject // fitness clips, unit-test crops). Higher-level callers can // also use ANSODBase::RunInference(input, vector, cam) // which crops each bbox and routes back here per crop. // ------------------------------------------------------------------ std::vector ANSONNXRTMPOSE::DetectObjects(const cv::Mat& inputImage, const std::string& camera_id) { try { // Snapshot engine handle + detector under brief lock; dereference // outside (mirrors ANSONNXYOLO::DetectObjects so concurrent // inference threads don't serialise on _mutex for the full // ORT Run() chain). std::shared_ptr engine; ANSONNXYOLO* personDetector = nullptr; // raw view; lifetime owned by m_personDetector std::string firstClass; { auto lk = TryLockWithTimeout("ANSONNXRTMPOSE::DetectObjects"); if (!lk.owns_lock()) return {}; if (!m_ortEngine) { _logger.LogError("ANSONNXRTMPOSE::DetectObjects", "ORT engine is null", __FILE__, __LINE__); return {}; } engine = m_ortEngine; // ref-count++ personDetector = m_personDetector.get(); if (!_classes.empty()) firstClass = _classes.front(); } // --- Top-down path --- if (personDetector) { std::vector detections = personDetector->RunInference(inputImage, camera_id); if (detections.empty()) { ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: detector found 0 detections cam='%s'", camera_id.c_str()); return {}; } // person.onnx may be a multi-class detector (e.g. a generic // COCO YOLO with 80 classes). The COCO 'person' class is id=0 // by convention. Drop everything else BEFORE pose / dedup so // we don't waste cycles on cars, dogs, signs, etc. std::vector persons; persons.reserve(detections.size()); for (auto& d : detections) { if (d.classId != 0) continue; if (d.box.width <= 0 || d.box.height <= 0) continue; persons.push_back(std::move(d)); } if (persons.empty()) { ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: %zu detections but 0 persons (classId==0) cam='%s'", detections.size(), camera_id.c_str()); return {}; } // IoU dedup — Ultralytics E2E NMS in person.onnx is too lax // (training-time IoU≈0.7 default), letting near-duplicate // boxes through. Drop duplicates BEFORE running pose so we // don't pay for two RTMPose inferences on the same subject. const size_t beforeDedup = persons.size(); persons = dedupByIoU(persons, kPersonDedupIoU); if (persons.size() != beforeDedup) { ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: dedup %zu -> %zu (IoU>%g) cam='%s'", beforeDedup, persons.size(), kPersonDedupIoU, camera_id.c_str()); } std::vector out; out.reserve(persons.size()); for (const auto& person : persons) { Object pose = engine->estimate(inputImage, person.box); if (pose.kps.empty()) continue; pose.cameraId = camera_id; pose.trackId = person.trackId; pose.confidence = person.confidence; pose.className = !person.className.empty() ? person.className : (firstClass.empty() ? std::string("person") : firstClass); out.push_back(std::move(pose)); } ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: %zu persons -> %zu poses cam='%s'", persons.size(), out.size(), camera_id.c_str()); return out; } // --- Standalone fallback path --- const cv::Rect wholeImage(0, 0, inputImage.cols, inputImage.rows); Object obj = engine->estimate(inputImage, wholeImage); if (obj.kps.empty()) return {}; obj.cameraId = camera_id; obj.className = firstClass.empty() ? std::string("person") : firstClass; std::vector out; out.push_back(std::move(obj)); ANS_DBG("ANSONNXRTMPOSE", "DetectObjects (standalone): kps=%d cam='%s'", engine->numKeypoints(), camera_id.c_str()); return out; } catch (const std::exception& e) { _logger.LogFatal("ANSONNXRTMPOSE::DetectObjects", e.what(), __FILE__, __LINE__); ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: EXCEPTION '%s'", e.what()); return {}; } } // ==================================================================== // TryLoadPersonDetector // // Look for person.onnx + person.names alongside the pose model. If // both exist, instantiate an ANSONNXYOLO front-end and load it via // LoadModelFromFolder (which creates the ORT session). On success // m_personDetector is non-null and DetectObjects switches to the // proper top-down pipeline. // // Missing files are NOT an error — the wrapper is designed to work // standalone for single-person test inputs. We log a debug line so // it's clear which path is active and return false. // ==================================================================== bool ANSONNXRTMPOSE::TryLoadPersonDetector(const std::string& modelFolder) { const std::string personOnnx = CreateFilePath(modelFolder, "person.onnx"); const std::string personNames = CreateFilePath(modelFolder, "person.names"); if (!FileExist(personOnnx) || !FileExist(personNames)) { ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: person.onnx/person.names not in '%s' " "-- standalone mode (whole-image pose)", modelFolder.c_str()); return false; } if (m_personDetector) { ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: already loaded, skipping"); return true; } try { auto detector = std::make_unique(); detector->SetLoadEngineOnCreation(this->_loadEngineOnCreation); ModelConfig personCfg{}; personCfg.detectionType = DetectionType::DETECTION; personCfg.modelType = ModelType::ONNXYOLO; personCfg.precisionType = PrecisionType::FP32; personCfg.inpHeight = 640; personCfg.inpWidth = 640; personCfg.detectionScoreThreshold = 0.5f; personCfg.modelConfThreshold = 0.5f; personCfg.modelMNSThreshold = 0.45f; personCfg.numKPS = 0; personCfg.gpuOptBatchSize = 1; personCfg.gpuMaxBatchSize = 1; std::string personLabelMap; const bool ok = detector->LoadModelFromFolder( /*licenseKey*/ _licenseKey, personCfg, /*modelName*/ "person", /*className*/ "person.names", /*modelFolder*/ modelFolder, personLabelMap); if (!ok) { _logger.LogWarn("ANSONNXRTMPOSE::TryLoadPersonDetector", "Person detector LoadModelFromFolder FAILED -- falling back to whole-image pose", __FILE__, __LINE__); ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: detector load FAILED"); return false; } m_personDetector = std::move(detector); ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: detector loaded OK from '%s' (labels='%s')", modelFolder.c_str(), personLabelMap.c_str()); return true; } catch (const std::exception& e) { _logger.LogWarn("ANSONNXRTMPOSE::TryLoadPersonDetector", e.what(), __FILE__, __LINE__); ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: EXCEPTION '%s'", e.what()); m_personDetector.reset(); return false; } } } // namespace ANSCENTER