Files

872 lines
39 KiB
C++
Raw Permalink Normal View History

2026-05-14 17:48:09 +10:00
#include "ANSONNXRTMPOSE.h"
#include "Utility.h"
#include "ANSLicense.h" // ANS_DBG macro for DebugView
#include <algorithm>
#include <cmath>
#include <chrono>
#include <sstream>
2026-05-14 19:17:09 +10:00
namespace {
// ── IoU dedup for end-to-end YOLO person detector output ──────────────
//
// Ultralytics E2E ONNX exports bake NMS into the graph, but its IoU
// threshold is set at export time (default ≈ 0.7) and we can't override
// it from outside. For a top-down pose pipeline that's too lax — pairs
// of overlapping bboxes survive and we end up running RTMPose twice on
// the same person.
//
// This filter walks the detector's output (already sorted by confidence
// in the E2E format), keeping each box only if its IoU with every
// already-kept box is below kPersonDedupIoU. O(N²) is fine because N is
// capped at 300 and typically <10 in practice.
constexpr float kPersonDedupIoU = 0.45f;
inline float boxIoU(const cv::Rect& a, const cv::Rect& b) {
const cv::Rect inter = a & b;
if (inter.area() <= 0) return 0.f;
const int uni = a.area() + b.area() - inter.area();
return uni > 0 ? static_cast<float>(inter.area()) / static_cast<float>(uni) : 0.f;
}
std::vector<ANSCENTER::Object> dedupByIoU(
const std::vector<ANSCENTER::Object>& persons, float iouThr) {
std::vector<ANSCENTER::Object> kept;
kept.reserve(persons.size());
for (const auto& p : persons) {
bool isDup = false;
for (const auto& k : kept) {
if (boxIoU(p.box, k.box) > iouThr) { isDup = true; break; }
}
if (!isDup) kept.push_back(p);
}
return kept;
}
} // anonymous namespace
2026-05-14 17:48:09 +10:00
namespace ANSCENTER {
// ====================================================================
// ONNXRtmPose — BasicOrtHandler subclass
// ====================================================================
ONNXRtmPose::ONNXRtmPose(const std::string& _onnx_path, unsigned int _num_threads)
: BasicOrtHandler(_onnx_path, _num_threads)
{
resolveInputShape();
}
ONNXRtmPose::ONNXRtmPose(const std::string& _onnx_path, EngineType engineType,
unsigned int _num_threads)
: BasicOrtHandler(_onnx_path, engineType, _num_threads)
{
resolveInputShape();
}
// ------------------------------------------------------------------
// resolveInputShape — read input/output dims off the model.
// Input : 1x3xHxW (RTMPose body-2d default 1x3x256x192)
// Output : simcc_x [1, K, ext_w], simcc_y [1, K, ext_h]
// The keypoint count K is read off the first output's dim[1].
// ------------------------------------------------------------------
void ONNXRtmPose::resolveInputShape() {
if (input_node_dims.size() >= 4) {
const int h = static_cast<int>(input_node_dims[2]);
const int w = static_cast<int>(input_node_dims[3]);
if (h > 0) m_input_h = h;
if (w > 0) m_input_w = w;
}
if (!output_node_dims.empty() && output_node_dims[0].size() >= 2) {
const int k = static_cast<int>(output_node_dims[0][1]);
if (k > 0) m_num_points = k;
}
ANS_DBG("ONNXRtmPose", "resolveInputShape: %dx%d kps=%d", m_input_w, m_input_h, m_num_points);
}
// ------------------------------------------------------------------
// GetAffineTransform — direct port of utils.h.
// ------------------------------------------------------------------
cv::Mat ONNXRtmPose::GetAffineTransform(float center_x, float center_y,
float scale_w, float scale_h,
int output_w, int output_h, bool inverse)
{
cv::Point2f src1(center_x, center_y);
cv::Point2f src2(center_x, center_y - scale_w * 0.5f);
cv::Point2f src3(src2.x - (src1.y - src2.y),
src2.y + (src1.x - src2.x));
const float dcx = output_w * 0.5f;
const float dcy = output_h * 0.5f;
cv::Point2f dst1(dcx, dcy);
cv::Point2f dst2(dcx, dcy - output_w * 0.5f);
cv::Point2f dst3(dst2.x - (dst1.y - dst2.y),
dst2.y + (dst1.x - dst2.x));
cv::Point2f srcPts[3] = { src1, src2, src3 };
cv::Point2f dstPts[3] = { dst1, dst2, dst3 };
return inverse ? cv::getAffineTransform(dstPts, srcPts)
: cv::getAffineTransform(srcPts, dstPts);
}
// ------------------------------------------------------------------
// CropImageByDetectBox — aspect-fit + 1.2x context expansion.
// Direct port of rtmpose_onnxruntime.cpp::CropImageByDetectBox.
// ------------------------------------------------------------------
void ONNXRtmPose::CropImageByDetectBox(const cv::Mat& image, const cv::Rect& box,
cv::Mat& crop, cv::Mat& affine_rev)
{
float w = static_cast<float>(box.width);
float h = static_cast<float>(box.height);
const float cx = box.x + w * 0.5f;
const float cy = box.y + h * 0.5f;
const float aspect = static_cast<float>(m_input_w) / static_cast<float>(m_input_h);
if (w > aspect * h) h = w / aspect;
else w = h * aspect;
const float scale_w = w * 1.2f;
const float scale_h = h * 1.2f;
cv::Mat affine = GetAffineTransform(cx, cy, scale_w, scale_h,
m_input_w, m_input_h, false);
affine_rev = GetAffineTransform(cx, cy, scale_w, scale_h,
m_input_w, m_input_h, true);
cv::warpAffine(image, crop, affine,
cv::Size(m_input_w, m_input_h), cv::INTER_LINEAR);
}
// ------------------------------------------------------------------
// transformInto — input is a 192x256 BGR person crop. Convert to
// RGB, ImageNet-normalize, lay out CHW into outBuffer, build the
// Ort::Value view over outBuffer.data(). Mirrors the design of
// ONNXYOLO::transformInto: per-call backing storage is owned by the
// caller so two threads sharing one ONNXRtmPose can preprocess and
// Run() concurrently without racing on input_values_handler.
// ------------------------------------------------------------------
Ort::Value ONNXRtmPose::transformInto(const cv::Mat& cropBGR,
std::vector<float>& outBuffer,
std::vector<int64_t>& outDims)
{
cv::Mat rgb;
cv::cvtColor(cropBGR, rgb, cv::COLOR_BGR2RGB);
cv::Mat floatImg;
rgb.convertTo(floatImg, CV_32FC3);
const int H = floatImg.rows;
const int W = floatImg.cols;
const size_t HW = static_cast<size_t>(H) * W;
outDims = { 1, 3, static_cast<int64_t>(H), static_cast<int64_t>(W) };
outBuffer.resize(3 * HW);
float* data = outBuffer.data();
// HWC -> CHW + (pixel - mean) / std. RGB channel order.
for (int y = 0; y < H; ++y) {
const float* row = floatImg.ptr<float>(y);
for (int x = 0; x < W; ++x) {
const float r = row[x * 3 + 0];
const float g = row[x * 3 + 1];
const float b = row[x * 3 + 2];
data[0 * HW + y * W + x] = (r - MEAN_R) / STD_R;
data[1 * HW + y * W + x] = (g - MEAN_G) / STD_G;
data[2 * HW + y * W + x] = (b - MEAN_B) / STD_B;
}
}
return Ort::Value::CreateTensor<float>(
*memory_info_handler,
outBuffer.data(),
outBuffer.size(),
outDims.data(),
outDims.size());
}
// BasicOrtHandler pure-virtual override. Thin shim that writes into
// the shared base members (legacy single-threaded caller path).
Ort::Value ONNXRtmPose::transform(const cv::Mat& mat)
{
Ort::Value v = transformInto(mat, input_values_handler, input_node_dims);
input_tensor_size = input_values_handler.size();
return v;
}
// RTMPose is single-person per call — batched inference doesn't apply.
Ort::Value ONNXRtmPose::transformBatch(const std::vector<cv::Mat>& /*images*/)
{
throw std::runtime_error("ONNXRtmPose::transformBatch: not supported (single-person model)");
}
// ------------------------------------------------------------------
// estimate — full per-bbox pipeline.
// ------------------------------------------------------------------
Object ONNXRtmPose::estimate(const cv::Mat& image, const cv::Rect& box)
{
Object obj{};
if (image.empty() || box.width <= 0 || box.height <= 0) {
ANS_DBG("ONNXRtmPose", "estimate: invalid input empty=%d box=%dx%d",
image.empty() ? 1 : 0, box.width, box.height);
return obj;
}
try {
// Affine crop -> 192x256 BGR person image, plus inverse-affine
// to map output coords back to original image.
cv::Mat crop, affine_reverse;
CropImageByDetectBox(image, box, crop, affine_reverse);
if (crop.empty()) {
ANS_DBG("ONNXRtmPose", "estimate: warpAffine produced empty crop");
return obj;
}
// Per-call backing storage so multiple threads sharing one
// ONNXRtmPose can Run() in parallel — see comment on
// transformInto.
std::vector<float> inputBuffer;
std::vector<int64_t> inputDims;
Ort::Value inputTensor = transformInto(crop, inputBuffer, inputDims);
auto outputTensors = ort_session->Run(
Ort::RunOptions{ nullptr },
input_node_names.data(),
&inputTensor, 1,
output_node_names.data(),
num_outputs);
if (outputTensors.size() < 2) {
ANS_DBG("ONNXRtmPose", "estimate: expected 2 outputs (simcc_x,simcc_y), got %zu",
outputTensors.size());
return obj;
}
// Decode SimCC. Disambiguate which output is X vs Y by length
// (X output's last dim is proportional to input W; Y's to H).
float* out0 = outputTensors[0].GetTensorMutableData<float>();
float* out1 = outputTensors[1].GetTensorMutableData<float>();
auto shape0 = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();
auto shape1 = outputTensors[1].GetTensorTypeAndShapeInfo().GetShape();
if (shape0.size() < 3 || shape1.size() < 3 ||
shape0[1] != shape1[1] || shape0[1] <= 0) {
ANS_DBG("ONNXRtmPose", "estimate: bad output shapes (rank or K mismatch)");
return obj;
}
const int joint_num = static_cast<int>(shape0[1]);
int W0 = static_cast<int>(shape0[2]);
int H1 = static_cast<int>(shape1[2]);
float* simcc_x = out0;
float* simcc_y = out1;
int extW = W0;
int extH = H1;
// If the first output looks H-shaped (closer to 2*input_h than
// 2*input_w), swap so simcc_x always indexes the W-axis.
const float ratioFirstW = (m_input_w > 0) ? float(W0) / float(m_input_w) : 0.f;
const float ratioFirstH = (m_input_h > 0) ? float(W0) / float(m_input_h) : 0.f;
if (std::abs(ratioFirstW - 2.f) > std::abs(ratioFirstH - 2.f)) {
std::swap(simcc_x, simcc_y);
extW = H1;
extH = W0;
}
const double m0 = affine_reverse.at<double>(0, 0);
const double m1 = affine_reverse.at<double>(0, 1);
const double m2 = affine_reverse.at<double>(0, 2);
const double m3 = affine_reverse.at<double>(1, 0);
const double m4 = affine_reverse.at<double>(1, 1);
const double m5 = affine_reverse.at<double>(1, 2);
const float imgW = static_cast<float>(image.cols);
const float imgH = static_cast<float>(image.rows);
std::vector<float> kps;
kps.reserve(static_cast<size_t>(joint_num) * 3);
std::vector<cv::Point2f> polygon;
polygon.reserve(joint_num);
std::stringstream xss, yss;
float meanScore = 0.f;
int meanCount = 0;
for (int i = 0; i < joint_num; ++i) {
const float* x_ptr = simcc_x + static_cast<size_t>(i) * extW;
const float* y_ptr = simcc_y + static_cast<size_t>(i) * extH;
const int px = static_cast<int>(std::max_element(x_ptr, x_ptr + extW) - x_ptr);
const int py = static_cast<int>(std::max_element(y_ptr, y_ptr + extH) - y_ptr);
const float score = std::max(x_ptr[px], y_ptr[py]);
// SimCC oversampling factor 2 — same constant as the reference impl.
const double raw_x = px * 0.5;
const double raw_y = py * 0.5;
float wx = static_cast<float>(m0 * raw_x + m1 * raw_y + m2);
float wy = static_cast<float>(m3 * raw_x + m4 * raw_y + m5);
wx = std::clamp(wx, 0.f, imgW - 1.f);
wy = std::clamp(wy, 0.f, imgH - 1.f);
kps.push_back(wx);
kps.push_back(wy);
kps.push_back(score);
polygon.emplace_back(wx, wy);
xss << wx;
yss << wy;
if (i + 1 < joint_num) { xss << ";"; yss << ";"; }
if (score > 0.f) { meanScore += score; ++meanCount; }
}
obj.classId = 0;
obj.confidence = (meanCount > 0) ? meanScore / meanCount : 0.f;
obj.box = box;
obj.kps = std::move(kps);
obj.polygon = std::move(polygon);
obj.extraInfo = xss.str() + "|" + yss.str();
return obj;
}
catch (const std::exception& e) {
ANS_DBG("ONNXRtmPose", "estimate: EXCEPTION '%s'", e.what());
return obj;
}
}
// ====================================================================
// ANSONNXRTMPOSE — ANSODBase wrapper
// ====================================================================
ANSONNXRTMPOSE::~ANSONNXRTMPOSE() {
try { Destroy(); }
catch (const std::exception& e) {
_logger.LogError("ANSONNXRTMPOSE::~ANSONNXRTMPOSE", e.what(), __FILE__, __LINE__);
}
}
bool ANSONNXRTMPOSE::Destroy() {
try {
2026-05-14 19:05:20 +10:00
if (m_personDetector) {
m_personDetector->Destroy();
m_personDetector.reset();
}
2026-05-14 17:48:09 +10:00
m_ortEngine.reset();
ANS_DBG("ANSONNXRTMPOSE", "Destroy: engine released");
return true;
}
catch (const std::exception& e) {
_logger.LogError("ANSONNXRTMPOSE::Destroy", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSONNXRTMPOSE::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) return false;
// ONNX models don't need an explicit precision-conversion pass here;
// the EP picks FP16 / FP32 at session-creation time. Mirrors
// ANSONNXYOLO::OptimizeModel which is also a no-op beyond folder
// resolution.
optimizedModelFolder = _modelFolder;
2026-05-14 19:05:20 +10:00
// Also bring up the bundled person-detector if present so its ORT
// session is constructed (and any cached engine is materialized)
// alongside the pose model. Mirrors the TRT counterpart's behavior.
(void)TryLoadPersonDetector(_modelFolder);
2026-05-14 17:48:09 +10:00
return true;
}
bool ANSONNXRTMPOSE::InitOrtEngine() {
try {
if (!FileExist(_modelFilePath)) {
_logger.LogError("ANSONNXRTMPOSE::InitOrtEngine",
"Model file does not exist: " + _modelFilePath, __FILE__, __LINE__);
ANS_DBG("ANSONNXRTMPOSE", "InitOrtEngine: missing onnx '%s'", _modelFilePath.c_str());
return false;
}
m_ortEngine = std::make_shared<ONNXRtmPose>(_modelFilePath);
ANS_DBG("ANSONNXRTMPOSE", "InitOrtEngine: ready %dx%d kps=%d",
m_ortEngine->inputWidth(), m_ortEngine->inputHeight(),
m_ortEngine->numKeypoints());
return true;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSONNXRTMPOSE::InitOrtEngine", e.what(), __FILE__, __LINE__);
ANS_DBG("ANSONNXRTMPOSE", "InitOrtEngine: EXCEPTION '%s'", e.what());
return false;
}
}
bool ANSONNXRTMPOSE::InitOrtEngine(ANSCENTER::EngineType engineType) {
try {
if (!FileExist(_modelFilePath)) {
_logger.LogError("ANSONNXRTMPOSE::InitOrtEngine",
"Model file does not exist: " + _modelFilePath, __FILE__, __LINE__);
return false;
}
m_ortEngine = std::make_shared<ONNXRtmPose>(_modelFilePath, engineType);
ANS_DBG("ANSONNXRTMPOSE", "InitOrtEngine(EP=%d): ready %dx%d kps=%d",
static_cast<int>(engineType),
m_ortEngine->inputWidth(), m_ortEngine->inputHeight(),
m_ortEngine->numKeypoints());
return true;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSONNXRTMPOSE::InitOrtEngine", e.what(), __FILE__, __LINE__);
return false;
}
}
// ------------------------------------------------------------------
// WarmUpEngine — single dummy inference to amortise first-call cost
// (CUDA arena alloc / cuDNN algo selection / DML shader compile).
// Non-fatal on failure: the engine is fully loaded before this runs.
// ------------------------------------------------------------------
void ANSONNXRTMPOSE::WarmUpEngine() {
if (!m_ortEngine) return;
const auto et = m_ortEngine->getEngineType();
if (et != EngineType::NVIDIA_GPU && et != EngineType::AMD_GPU) {
ANS_DBG("ANSONNXRTMPOSE", "Warm-up skipped (CPU/OpenVINO EP)");
return;
}
try {
const int W = m_ortEngine->inputWidth();
const int H = m_ortEngine->inputHeight();
cv::Mat dummy(H, W, CV_8UC3, cv::Scalar(128, 128, 128));
const cv::Rect wholeImage(0, 0, W, H);
auto t0 = std::chrono::steady_clock::now();
(void)m_ortEngine->estimate(dummy, wholeImage);
auto t1 = std::chrono::steady_clock::now();
auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0).count();
ANS_DBG("ANSONNXRTMPOSE", "Warm-up done: %lld ms (%dx%d)", (long long)ms, W, H);
}
catch (const std::exception& e) {
_logger.LogWarn("ANSONNXRTMPOSE::WarmUpEngine",
std::string("Warm-up skipped (non-fatal): ") + e.what(),
__FILE__, __LINE__);
ANS_DBG("ANSONNXRTMPOSE", "Warm-up: EXCEPTION '%s'", e.what());
}
catch (...) {
_logger.LogWarn("ANSONNXRTMPOSE::WarmUpEngine",
"Warm-up skipped (unknown exception)", __FILE__, __LINE__);
}
}
bool ANSONNXRTMPOSE::Initialize(std::string licenseKey, ModelConfig modelConfig,
const std::string& modelZipFilePath,
const std::string& modelZipPassword,
std::string& labelMap)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
ModelLoadingGuard mlg(_modelLoading);
ANS_DBG("ANSONNXRTMPOSE", "Initialize: zip='%s'", modelZipFilePath.c_str());
try {
_modelLoadValid = false;
bool result = ANSODBase::Initialize(licenseKey, modelConfig,
modelZipFilePath, modelZipPassword, labelMap);
if (!result) return false;
_modelConfig = modelConfig;
_modelConfig.modelType = ModelType::ONNXRTMPOSE;
if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = 256;
if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = 192;
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133)
_modelConfig.numKPS = 26;
if (_modelConfig.kpsThreshold <= 0)
_modelConfig.kpsThreshold = KPS_THRESHOLD;
KPS_THRESHOLD = _modelConfig.kpsThreshold;
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1];
}
}
else {
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
std::ifstream isValid(_classFilePath);
if (!isValid) LoadClassesFromString();
else LoadClassesFromFile();
}
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
if (this->_loadEngineOnCreation) {
if (!InitOrtEngine()) {
_logger.LogError("ANSONNXRTMPOSE::Initialize",
"Failed to create ONNX Runtime engine: " + _modelFilePath,
__FILE__, __LINE__);
return false;
}
// Sync ModelConfig.numKPS to whatever the model actually has,
// so consumers reading GetModelConfig() see the truth.
if (m_ortEngine && m_ortEngine->numKeypoints() > 0)
_modelConfig.numKPS = m_ortEngine->numKeypoints();
WarmUpEngine();
}
2026-05-14 19:05:20 +10:00
// Top-down pipeline: try to load the bundled person detector.
// Falls back gracefully (whole-image pose) when the files are missing.
(void)TryLoadPersonDetector(_modelFolder);
2026-05-14 17:48:09 +10:00
_modelLoadValid = true;
_isInitialized = true;
return true;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSONNXRTMPOSE::Initialize", e.what(), __FILE__, __LINE__);
ANS_DBG("ANSONNXRTMPOSE", "Initialize: EXCEPTION '%s'", e.what());
return false;
}
}
bool ANSONNXRTMPOSE::LoadModel(const std::string& modelZipFilePath,
const std::string& modelZipPassword)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
ModelLoadingGuard mlg(_modelLoading);
ANS_DBG("ANSONNXRTMPOSE", "LoadModel: zip='%s'", modelZipFilePath.c_str());
try {
bool result = ANSODBase::LoadModel(modelZipFilePath, modelZipPassword);
if (!result) return false;
_modelConfig.modelType = ModelType::ONNXRTMPOSE;
if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = 256;
if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = 192;
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133)
_modelConfig.numKPS = 26;
if (_modelConfig.kpsThreshold <= 0)
_modelConfig.kpsThreshold = KPS_THRESHOLD;
KPS_THRESHOLD = _modelConfig.kpsThreshold;
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1];
}
}
else {
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
std::ifstream isValid(_classFilePath);
if (!isValid) LoadClassesFromString();
else LoadClassesFromFile();
}
if (this->_loadEngineOnCreation) {
if (!InitOrtEngine()) { _modelLoadValid = false; return false; }
if (m_ortEngine && m_ortEngine->numKeypoints() > 0)
_modelConfig.numKPS = m_ortEngine->numKeypoints();
WarmUpEngine();
}
2026-05-14 19:05:20 +10:00
(void)TryLoadPersonDetector(_modelFolder);
2026-05-14 17:48:09 +10:00
_modelLoadValid = true;
_isInitialized = true;
return true;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSONNXRTMPOSE::LoadModel", e.what(), __FILE__, __LINE__);
ANS_DBG("ANSONNXRTMPOSE", "LoadModel: EXCEPTION '%s'", e.what());
return false;
}
}
bool ANSONNXRTMPOSE::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig,
std::string modelName, std::string className,
const std::string& modelFolder,
std::string& labelMap)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
ModelLoadingGuard mlg(_modelLoading);
ANS_DBG("ANSONNXRTMPOSE", "LoadModelFromFolder: folder='%s' model='%s'",
modelFolder.c_str(), modelName.c_str());
try {
bool result = ANSODBase::LoadModelFromFolder(licenseKey, modelConfig,
modelName, className,
modelFolder, labelMap);
if (!result) return false;
_modelConfig = modelConfig;
_modelConfig.modelType = ModelType::ONNXRTMPOSE;
if (_modelConfig.inpHeight <= 0) _modelConfig.inpHeight = 256;
if (_modelConfig.inpWidth <= 0) _modelConfig.inpWidth = 192;
if (_modelConfig.numKPS <= 0 || _modelConfig.numKPS > 133)
_modelConfig.numKPS = 26;
if (_modelConfig.kpsThreshold <= 0)
_modelConfig.kpsThreshold = KPS_THRESHOLD;
KPS_THRESHOLD = _modelConfig.kpsThreshold;
std::string _modelName = modelName;
if (_modelName.empty()) _modelName = "train_last";
std::string modelFullName = _modelName + ".onnx";
_modelFilePath = CreateFilePath(_modelFolder, modelFullName);
if (FileExist(_modelConfigFile)) {
ModelType modelType;
std::vector<int> inputShape;
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
if (inputShape.size() == 2) {
if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0];
if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1];
}
}
else {
_classFilePath = CreateFilePath(_modelFolder, className);
std::ifstream isValid(_classFilePath);
if (!isValid) LoadClassesFromString();
else LoadClassesFromFile();
}
labelMap.clear();
if (!_classes.empty())
labelMap = VectorToCommaSeparatedString(_classes);
if (this->_loadEngineOnCreation) {
if (!InitOrtEngine()) { _modelLoadValid = false; return false; }
if (m_ortEngine && m_ortEngine->numKeypoints() > 0)
_modelConfig.numKPS = m_ortEngine->numKeypoints();
WarmUpEngine();
}
2026-05-14 19:05:20 +10:00
(void)TryLoadPersonDetector(_modelFolder);
2026-05-14 17:48:09 +10:00
_modelLoadValid = true;
_isInitialized = true;
return true;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSONNXRTMPOSE::LoadModelFromFolder", e.what(), __FILE__, __LINE__);
ANS_DBG("ANSONNXRTMPOSE", "LoadModelFromFolder: EXCEPTION '%s'", e.what());
return false;
}
}
std::vector<Object> ANSONNXRTMPOSE::RunInference(const cv::Mat& inputImgBGR) {
return RunInference(inputImgBGR, "");
}
std::vector<Object> ANSONNXRTMPOSE::RunInference(const cv::Mat& inputImgBGR,
const std::string& camera_id)
{
if (_modelLoading.load()) return {};
{
auto lock = TryLockWithTimeout("ANSONNXRTMPOSE::RunInference");
if (!lock.owns_lock()) return {};
if (!_modelLoadValid) {
_logger.LogError("ANSONNXRTMPOSE::RunInference", "Model not loaded", __FILE__, __LINE__);
return {};
}
if (!_licenseValid) {
_logger.LogError("ANSONNXRTMPOSE::RunInference", "Invalid license", __FILE__, __LINE__);
return {};
}
if (!_isInitialized) {
_logger.LogError("ANSONNXRTMPOSE::RunInference", "Model not initialized", __FILE__, __LINE__);
return {};
}
if (inputImgBGR.empty() || inputImgBGR.cols < 10 || inputImgBGR.rows < 10)
return {};
}
try { return DetectObjects(inputImgBGR, camera_id); }
catch (const std::exception& e) {
_logger.LogFatal("ANSONNXRTMPOSE::RunInference", e.what(), __FILE__, __LINE__);
ANS_DBG("ANSONNXRTMPOSE", "RunInference: EXCEPTION '%s'", e.what());
return {};
}
}
// ------------------------------------------------------------------
2026-05-14 19:05:20 +10:00
// DetectObjects — two pipelines:
//
// (a) Top-down (preferred) — m_personDetector loaded from the
// bundled person.onnx + person.names. Detect people first,
// then run RTMPose ONCE per person bbox. This is what RTMPose
// is designed for; works on multi-person scenes with small /
// distant subjects.
//
// (b) Standalone fallback — no detector files in the model folder.
// Treat the whole image as a single person crop. Only sensible
// for inputs that are already a person (selfies, single-subject
// fitness clips, unit-test crops). Higher-level callers can
// also use ANSODBase::RunInference(input, vector<Rect>, cam)
// which crops each bbox and routes back here per crop.
2026-05-14 17:48:09 +10:00
// ------------------------------------------------------------------
std::vector<Object> ANSONNXRTMPOSE::DetectObjects(const cv::Mat& inputImage,
const std::string& camera_id)
{
try {
2026-05-14 19:05:20 +10:00
// Snapshot engine handle + detector under brief lock; dereference
// outside (mirrors ANSONNXYOLO::DetectObjects so concurrent
// inference threads don't serialise on _mutex for the full
// ORT Run() chain).
2026-05-14 17:48:09 +10:00
std::shared_ptr<ONNXRtmPose> engine;
2026-05-14 19:05:20 +10:00
ANSONNXYOLO* personDetector = nullptr; // raw view; lifetime owned by m_personDetector
std::string firstClass;
2026-05-14 17:48:09 +10:00
{
auto lk = TryLockWithTimeout("ANSONNXRTMPOSE::DetectObjects");
if (!lk.owns_lock()) return {};
if (!m_ortEngine) {
_logger.LogError("ANSONNXRTMPOSE::DetectObjects",
"ORT engine is null", __FILE__, __LINE__);
return {};
}
2026-05-14 19:05:20 +10:00
engine = m_ortEngine; // ref-count++
personDetector = m_personDetector.get();
2026-05-14 17:48:09 +10:00
if (!_classes.empty()) firstClass = _classes.front();
}
2026-05-14 19:05:20 +10:00
// --- Top-down path ---
if (personDetector) {
std::vector<Object> detections = personDetector->RunInference(inputImage, camera_id);
if (detections.empty()) {
ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: detector found 0 detections cam='%s'",
2026-05-14 19:05:20 +10:00
camera_id.c_str());
return {};
}
// person.onnx may be a multi-class detector (e.g. a generic
// COCO YOLO with 80 classes). The COCO 'person' class is id=0
// by convention. Drop everything else BEFORE pose / dedup so
// we don't waste cycles on cars, dogs, signs, etc.
std::vector<Object> persons;
persons.reserve(detections.size());
for (auto& d : detections) {
if (d.classId != 0) continue;
if (d.box.width <= 0 || d.box.height <= 0) continue;
persons.push_back(std::move(d));
}
if (persons.empty()) {
ANS_DBG("ANSONNXRTMPOSE",
"DetectObjects: %zu detections but 0 persons (classId==0) cam='%s'",
detections.size(), camera_id.c_str());
return {};
}
2026-05-14 19:17:09 +10:00
// IoU dedup — Ultralytics E2E NMS in person.onnx is too lax
// (training-time IoU≈0.7 default), letting near-duplicate
// boxes through. Drop duplicates BEFORE running pose so we
// don't pay for two RTMPose inferences on the same subject.
const size_t beforeDedup = persons.size();
persons = dedupByIoU(persons, kPersonDedupIoU);
if (persons.size() != beforeDedup) {
ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: dedup %zu -> %zu (IoU>%g) cam='%s'",
beforeDedup, persons.size(), kPersonDedupIoU, camera_id.c_str());
}
2026-05-14 19:05:20 +10:00
std::vector<Object> out;
out.reserve(persons.size());
for (const auto& person : persons) {
Object pose = engine->estimate(inputImage, person.box);
if (pose.kps.empty()) continue;
pose.cameraId = camera_id;
pose.trackId = person.trackId;
pose.confidence = person.confidence;
pose.className = !person.className.empty() ? person.className
: (firstClass.empty() ? std::string("person") : firstClass);
out.push_back(std::move(pose));
}
ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: %zu persons -> %zu poses cam='%s'",
persons.size(), out.size(), camera_id.c_str());
return out;
}
// --- Standalone fallback path ---
2026-05-14 17:48:09 +10:00
const cv::Rect wholeImage(0, 0, inputImage.cols, inputImage.rows);
Object obj = engine->estimate(inputImage, wholeImage);
if (obj.kps.empty()) return {};
obj.cameraId = camera_id;
obj.className = firstClass.empty() ? std::string("person") : firstClass;
std::vector<Object> out;
out.push_back(std::move(obj));
2026-05-14 19:05:20 +10:00
ANS_DBG("ANSONNXRTMPOSE", "DetectObjects (standalone): kps=%d cam='%s'",
2026-05-14 17:48:09 +10:00
engine->numKeypoints(), camera_id.c_str());
return out;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSONNXRTMPOSE::DetectObjects", e.what(), __FILE__, __LINE__);
ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: EXCEPTION '%s'", e.what());
return {};
}
}
2026-05-14 19:05:20 +10:00
// ====================================================================
// TryLoadPersonDetector
//
// Look for person.onnx + person.names alongside the pose model. If
// both exist, instantiate an ANSONNXYOLO front-end and load it via
// LoadModelFromFolder (which creates the ORT session). On success
// m_personDetector is non-null and DetectObjects switches to the
// proper top-down pipeline.
//
// Missing files are NOT an error — the wrapper is designed to work
// standalone for single-person test inputs. We log a debug line so
// it's clear which path is active and return false.
// ====================================================================
bool ANSONNXRTMPOSE::TryLoadPersonDetector(const std::string& modelFolder) {
const std::string personOnnx = CreateFilePath(modelFolder, "person.onnx");
const std::string personNames = CreateFilePath(modelFolder, "person.names");
if (!FileExist(personOnnx) || !FileExist(personNames)) {
ANS_DBG("ANSONNXRTMPOSE",
"TryLoadPersonDetector: person.onnx/person.names not in '%s' "
"-- standalone mode (whole-image pose)", modelFolder.c_str());
return false;
}
if (m_personDetector) {
ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: already loaded, skipping");
return true;
}
try {
auto detector = std::make_unique<ANSONNXYOLO>();
detector->SetLoadEngineOnCreation(this->_loadEngineOnCreation);
ModelConfig personCfg{};
personCfg.detectionType = DetectionType::DETECTION;
personCfg.modelType = ModelType::ONNXYOLO;
personCfg.precisionType = PrecisionType::FP32;
personCfg.inpHeight = 640;
personCfg.inpWidth = 640;
personCfg.detectionScoreThreshold = 0.5f;
personCfg.modelConfThreshold = 0.5f;
personCfg.modelMNSThreshold = 0.45f;
personCfg.numKPS = 0;
personCfg.gpuOptBatchSize = 1;
personCfg.gpuMaxBatchSize = 1;
std::string personLabelMap;
const bool ok = detector->LoadModelFromFolder(
/*licenseKey*/ _licenseKey,
personCfg,
/*modelName*/ "person",
/*className*/ "person.names",
/*modelFolder*/ modelFolder,
personLabelMap);
if (!ok) {
_logger.LogWarn("ANSONNXRTMPOSE::TryLoadPersonDetector",
"Person detector LoadModelFromFolder FAILED -- falling back to whole-image pose",
__FILE__, __LINE__);
ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: detector load FAILED");
return false;
}
m_personDetector = std::move(detector);
ANS_DBG("ANSONNXRTMPOSE",
"TryLoadPersonDetector: detector loaded OK from '%s' (labels='%s')",
modelFolder.c_str(), personLabelMap.c_str());
return true;
}
catch (const std::exception& e) {
_logger.LogWarn("ANSONNXRTMPOSE::TryLoadPersonDetector", e.what(), __FILE__, __LINE__);
ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: EXCEPTION '%s'", e.what());
m_personDetector.reset();
return false;
}
}
2026-05-14 17:48:09 +10:00
} // namespace ANSCENTER