Add person model into RTMPOSE

This commit is contained in:
2026-05-14 19:05:20 +10:00
parent ee8f0d1163
commit 8e4d9a9e5d
5 changed files with 478 additions and 36 deletions
+144 -11
View File
@@ -300,6 +300,10 @@ namespace ANSCENTER {
bool ANSONNXRTMPOSE::Destroy() {
try {
if (m_personDetector) {
m_personDetector->Destroy();
m_personDetector.reset();
}
m_ortEngine.reset();
ANS_DBG("ANSONNXRTMPOSE", "Destroy: engine released");
return true;
@@ -317,6 +321,11 @@ namespace ANSCENTER {
// ANSONNXYOLO::OptimizeModel which is also a no-op beyond folder
// resolution.
optimizedModelFolder = _modelFolder;
// Also bring up the bundled person-detector if present so its ORT
// session is constructed (and any cached engine is materialized)
// alongside the pose model. Mirrors the TRT counterpart's behavior.
(void)TryLoadPersonDetector(_modelFolder);
return true;
}
@@ -457,6 +466,10 @@ namespace ANSCENTER {
WarmUpEngine();
}
// Top-down pipeline: try to load the bundled person detector.
// Falls back gracefully (whole-image pose) when the files are missing.
(void)TryLoadPersonDetector(_modelFolder);
_modelLoadValid = true;
_isInitialized = true;
return true;
@@ -510,6 +523,9 @@ namespace ANSCENTER {
_modelConfig.numKPS = m_ortEngine->numKeypoints();
WarmUpEngine();
}
(void)TryLoadPersonDetector(_modelFolder);
_modelLoadValid = true;
_isInitialized = true;
return true;
@@ -577,6 +593,9 @@ namespace ANSCENTER {
_modelConfig.numKPS = m_ortEngine->numKeypoints();
WarmUpEngine();
}
(void)TryLoadPersonDetector(_modelFolder);
_modelLoadValid = true;
_isInitialized = true;
return true;
@@ -623,21 +642,32 @@ namespace ANSCENTER {
}
// ------------------------------------------------------------------
// DetectObjects — top-down: with no external bbox, treat the whole
// input image as the person crop and run RTMPose. Higher-level
// orchestration that already has per-person bboxes should call
// ANSODBase::RunInference(input, vector<Rect>, cam) — that overload
// crops each bbox and routes back here per crop.
// DetectObjects — two pipelines:
//
// (a) Top-down (preferred) — m_personDetector loaded from the
// bundled person.onnx + person.names. Detect people first,
// then run RTMPose ONCE per person bbox. This is what RTMPose
// is designed for; works on multi-person scenes with small /
// distant subjects.
//
// (b) Standalone fallback — no detector files in the model folder.
// Treat the whole image as a single person crop. Only sensible
// for inputs that are already a person (selfies, single-subject
// fitness clips, unit-test crops). Higher-level callers can
// also use ANSODBase::RunInference(input, vector<Rect>, cam)
// which crops each bbox and routes back here per crop.
// ------------------------------------------------------------------
std::vector<Object> ANSONNXRTMPOSE::DetectObjects(const cv::Mat& inputImage,
const std::string& camera_id)
{
try {
// Snapshot engine handle under brief lock; dereference outside
// (mirrors ANSONNXYOLO::DetectObjects so concurrent inference
// threads don't serialise on _mutex for the full ORT Run()).
// Snapshot engine handle + detector under brief lock; dereference
// outside (mirrors ANSONNXYOLO::DetectObjects so concurrent
// inference threads don't serialise on _mutex for the full
// ORT Run() chain).
std::shared_ptr<ONNXRtmPose> engine;
std::string firstClass;
ANSONNXYOLO* personDetector = nullptr; // raw view; lifetime owned by m_personDetector
std::string firstClass;
{
auto lk = TryLockWithTimeout("ANSONNXRTMPOSE::DetectObjects");
if (!lk.owns_lock()) return {};
@@ -646,10 +676,39 @@ namespace ANSCENTER {
"ORT engine is null", __FILE__, __LINE__);
return {};
}
engine = m_ortEngine; // ref-count++
engine = m_ortEngine; // ref-count++
personDetector = m_personDetector.get();
if (!_classes.empty()) firstClass = _classes.front();
}
// --- Top-down path ---
if (personDetector) {
std::vector<Object> persons = personDetector->RunInference(inputImage, camera_id);
if (persons.empty()) {
ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: detector found 0 persons cam='%s'",
camera_id.c_str());
return {};
}
std::vector<Object> out;
out.reserve(persons.size());
for (const auto& person : persons) {
if (person.box.width <= 0 || person.box.height <= 0) continue;
Object pose = engine->estimate(inputImage, person.box);
if (pose.kps.empty()) continue;
pose.cameraId = camera_id;
pose.trackId = person.trackId;
pose.confidence = person.confidence;
pose.className = !person.className.empty() ? person.className
: (firstClass.empty() ? std::string("person") : firstClass);
out.push_back(std::move(pose));
}
ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: %zu persons -> %zu poses cam='%s'",
persons.size(), out.size(), camera_id.c_str());
return out;
}
// --- Standalone fallback path ---
const cv::Rect wholeImage(0, 0, inputImage.cols, inputImage.rows);
Object obj = engine->estimate(inputImage, wholeImage);
if (obj.kps.empty()) return {};
@@ -659,7 +718,7 @@ namespace ANSCENTER {
std::vector<Object> out;
out.push_back(std::move(obj));
ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: kps=%d cam='%s'",
ANS_DBG("ANSONNXRTMPOSE", "DetectObjects (standalone): kps=%d cam='%s'",
engine->numKeypoints(), camera_id.c_str());
return out;
}
@@ -670,4 +729,78 @@ namespace ANSCENTER {
}
}
// ====================================================================
// TryLoadPersonDetector
//
// Look for person.onnx + person.names alongside the pose model. If
// both exist, instantiate an ANSONNXYOLO front-end and load it via
// LoadModelFromFolder (which creates the ORT session). On success
// m_personDetector is non-null and DetectObjects switches to the
// proper top-down pipeline.
//
// Missing files are NOT an error — the wrapper is designed to work
// standalone for single-person test inputs. We log a debug line so
// it's clear which path is active and return false.
// ====================================================================
bool ANSONNXRTMPOSE::TryLoadPersonDetector(const std::string& modelFolder) {
const std::string personOnnx = CreateFilePath(modelFolder, "person.onnx");
const std::string personNames = CreateFilePath(modelFolder, "person.names");
if (!FileExist(personOnnx) || !FileExist(personNames)) {
ANS_DBG("ANSONNXRTMPOSE",
"TryLoadPersonDetector: person.onnx/person.names not in '%s' "
"-- standalone mode (whole-image pose)", modelFolder.c_str());
return false;
}
if (m_personDetector) {
ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: already loaded, skipping");
return true;
}
try {
auto detector = std::make_unique<ANSONNXYOLO>();
detector->SetLoadEngineOnCreation(this->_loadEngineOnCreation);
ModelConfig personCfg{};
personCfg.detectionType = DetectionType::DETECTION;
personCfg.modelType = ModelType::ONNXYOLO;
personCfg.precisionType = PrecisionType::FP32;
personCfg.inpHeight = 640;
personCfg.inpWidth = 640;
personCfg.detectionScoreThreshold = 0.5f;
personCfg.modelConfThreshold = 0.5f;
personCfg.modelMNSThreshold = 0.45f;
personCfg.numKPS = 0;
personCfg.gpuOptBatchSize = 1;
personCfg.gpuMaxBatchSize = 1;
std::string personLabelMap;
const bool ok = detector->LoadModelFromFolder(
/*licenseKey*/ _licenseKey,
personCfg,
/*modelName*/ "person",
/*className*/ "person.names",
/*modelFolder*/ modelFolder,
personLabelMap);
if (!ok) {
_logger.LogWarn("ANSONNXRTMPOSE::TryLoadPersonDetector",
"Person detector LoadModelFromFolder FAILED -- falling back to whole-image pose",
__FILE__, __LINE__);
ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: detector load FAILED");
return false;
}
m_personDetector = std::move(detector);
ANS_DBG("ANSONNXRTMPOSE",
"TryLoadPersonDetector: detector loaded OK from '%s' (labels='%s')",
modelFolder.c_str(), personLabelMap.c_str());
return true;
}
catch (const std::exception& e) {
_logger.LogWarn("ANSONNXRTMPOSE::TryLoadPersonDetector", e.what(), __FILE__, __LINE__);
ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: EXCEPTION '%s'", e.what());
m_personDetector.reset();
return false;
}
}
} // namespace ANSCENTER
+16
View File
@@ -3,6 +3,7 @@
#pragma once
#include "ANSEngineCommon.h"
#include "ONNXEngine.h"
#include "ANSONNXYOLO.h" // internal person-detector front-end (top-down pipeline)
namespace ANSCENTER {
@@ -144,6 +145,21 @@ namespace ANSCENTER {
// member here on another thread.
std::shared_ptr<ONNXRtmPose> m_ortEngine;
// ── Optional internal person detector (top-down pipeline) ───────
// RTMPose is a top-down single-person estimator. If the model
// folder bundles person.onnx + person.names alongside the pose
// model, we auto-load an ANSONNXYOLO person detector and run pose
// ONCE PER DETECTED PERSON. When those files are missing the
// wrapper falls back to whole-image pose so single-person test
// inputs (selfies, fitness clips) still work.
std::unique_ptr<ANSONNXYOLO> m_personDetector;
// Look for person.onnx + person.names in `modelFolder`. Loads
// m_personDetector and returns true on success; returns false
// (without logging an error) if either file is missing — caller
// should treat that as "no detector, fall back to whole-image".
bool TryLoadPersonDetector(const std::string& modelFolder);
bool InitOrtEngine();
bool InitOrtEngine(ANSCENTER::EngineType engineType);
void WarmUpEngine();
+150 -5
View File
@@ -46,6 +46,12 @@ namespace ANSCENTER {
return false;
}
ANS_DBG("ANSRTRTMPOSE", "OptimizeModel: build OK -> '%s'", optimizedModelFolder.c_str());
// Also build the bundled person-detector engine if present.
// TryLoadPersonDetector triggers ANSRTYOLO::LoadModelFromFolder
// which performs the engine build (= optimization) and caches
// it to disk; the next Initialize/LoadModel will hit the cache.
(void)TryLoadPersonDetector(_modelFolder);
return true;
}
catch (std::exception& e) {
@@ -138,6 +144,11 @@ namespace ANSCENTER {
ANS_DBG("ANSRTRTMPOSE", "LoadModel: engine ready %dx%d kps=%d sharedPool=%d",
RTM_INPUT_W, RTM_INPUT_H, _modelConfig.numKPS, m_usingSharedPool ? 1 : 0);
}
// Top-down pipeline: try to load the bundled person detector.
// Falls back gracefully (whole-image pose) when the files are missing.
(void)TryLoadPersonDetector(_modelFolder);
_modelLoadValid = true;
_isInitialized = true;
return true;
@@ -242,6 +253,9 @@ namespace ANSCENTER {
ANS_DBG("ANSRTRTMPOSE", "LoadModelFromFolder: engine ready %dx%d kps=%d sharedPool=%d",
RTM_INPUT_W, RTM_INPUT_H, _modelConfig.numKPS, m_usingSharedPool ? 1 : 0);
}
(void)TryLoadPersonDetector(_modelFolder);
_modelLoadValid = true;
_isInitialized = true;
return true;
@@ -344,6 +358,9 @@ namespace ANSCENTER {
ANS_DBG("ANSRTRTMPOSE", "Initialize: engine ready %dx%d kps=%d sharedPool=%d",
RTM_INPUT_W, RTM_INPUT_H, _modelConfig.numKPS, m_usingSharedPool ? 1 : 0);
}
(void)TryLoadPersonDetector(_modelFolder);
_modelLoadValid = true;
_isInitialized = true;
return true;
@@ -370,11 +387,56 @@ namespace ANSCENTER {
return {};
}
try {
// Top-down pose estimator: with no external bbox, treat the whole
// image as a single person crop. Higher-level orchestration that
// wants per-person pose should call ANSODBase::RunInference(input,
// bboxes, cam) instead — that overload crops each bbox and routes
// back through us per crop.
// Two pipelines:
//
// (a) Top-down (preferred) — m_personDetector loaded from
// the bundled person.onnx + person.names. Detect people
// first, then run RTMPose ONCE per person bbox. This is
// what RTMPose is designed for; works on multi-person
// scenes with small / distant subjects.
//
// (b) Standalone fallback — no detector files in the model
// folder. Treat the whole image as a single person crop.
// Only sensible for inputs that are already a person
// (selfies, single-subject fitness clips, unit-test
// crops). Higher-level callers can also use the
// ANSODBase::RunInference(input, vector<Rect>, cam)
// overload which crops each bbox and routes per-crop
// through our Preprocess.
if (m_personDetector) {
std::vector<Object> persons = m_personDetector->RunInference(inputImgBGR, camera_id);
if (persons.empty()) {
ANS_DBG("ANSRTRTMPOSE", "RunInference: detector found 0 persons cam='%s'",
camera_id.c_str());
return {};
}
std::vector<Object> results;
results.reserve(persons.size());
for (const auto& person : persons) {
// person.onnx is a single-class model — accept anything
// the detector returned, but defend against degenerate
// boxes (zero/negative area, off-image).
if (person.box.width <= 0 || person.box.height <= 0) continue;
auto poseResults = InferenceOnBox(inputImgBGR, person.box, camera_id);
for (auto& obj : poseResults) {
// Carry the detector's track-id and confidence so
// downstream tracking/visualization stays coherent
// across frames. RTMPose itself has no class /
// tracking concept of its own.
obj.trackId = person.trackId;
obj.confidence = person.confidence;
if (!person.className.empty()) obj.className = person.className;
results.push_back(std::move(obj));
}
}
ANS_DBG("ANSRTRTMPOSE", "RunInference: %zu persons -> %zu poses cam='%s'",
persons.size(), results.size(), camera_id.c_str());
return results;
}
// Fallback — single-person whole-image pose
const cv::Rect wholeImage(0, 0, inputImgBGR.cols, inputImgBGR.rows);
return InferenceOnBox(inputImgBGR, wholeImage, camera_id);
}
@@ -394,6 +456,10 @@ namespace ANSCENTER {
bool ANSRTRTMPOSE::Destroy() {
try {
if (m_personDetector) {
m_personDetector->Destroy();
m_personDetector.reset();
}
if (m_usingSharedPool) {
EnginePoolManager<float>::instance().release(m_poolKey);
m_trtEngine.reset();
@@ -699,4 +765,83 @@ namespace ANSCENTER {
cv::Size(RTM_INPUT_W, RTM_INPUT_H), cv::INTER_LINEAR);
}
// ====================================================================
// TryLoadPersonDetector
//
// Look for person.onnx + person.names alongside the pose model. If
// both exist, instantiate an ANSRTYOLO front-end and load it via
// LoadModelFromFolder (which builds / loads the TRT engine from
// cache). On success m_personDetector is non-null and RunInference
// switches to the proper top-down pipeline.
//
// Missing files are NOT an error — the wrapper is designed to work
// standalone for single-person test inputs. We log a debug line so
// it's clear which path is active and return false.
// ====================================================================
bool ANSRTRTMPOSE::TryLoadPersonDetector(const std::string& modelFolder) {
const std::string personOnnx = CreateFilePath(modelFolder, "person.onnx");
const std::string personNames = CreateFilePath(modelFolder, "person.names");
if (!FileExist(personOnnx) || !FileExist(personNames)) {
ANS_DBG("ANSRTRTMPOSE",
"TryLoadPersonDetector: person.onnx/person.names not in '%s' "
"-- standalone mode (whole-image pose)", modelFolder.c_str());
return false;
}
if (m_personDetector) {
ANS_DBG("ANSRTRTMPOSE", "TryLoadPersonDetector: already loaded, skipping");
return true;
}
try {
auto detector = std::make_unique<ANSRTYOLO>();
detector->SetLoadEngineOnCreation(this->_loadEngineOnCreation);
// SetMaxSlotsPerGpu is declared private in ANSRTYOLO (it's a
// private override of ANSODBase's public virtual). Call it via
// the base-class interface where access is public.
static_cast<ANSODBase*>(detector.get())->SetMaxSlotsPerGpu(m_maxSlotsPerGpu);
ModelConfig personCfg{};
personCfg.detectionType = DetectionType::DETECTION;
personCfg.modelType = ModelType::RTYOLO;
personCfg.precisionType = PrecisionType::FP16;
personCfg.inpHeight = 640;
personCfg.inpWidth = 640;
personCfg.detectionScoreThreshold = 0.5f;
personCfg.modelConfThreshold = 0.5f;
personCfg.modelMNSThreshold = 0.45f;
personCfg.numKPS = 0;
personCfg.gpuOptBatchSize = 1;
personCfg.gpuMaxBatchSize = 1;
personCfg.gpuDeviceIndex = m_options.deviceIndex;
std::string personLabelMap;
const bool ok = detector->LoadModelFromFolder(
/*licenseKey*/ _licenseKey,
personCfg,
/*modelName*/ "person",
/*className*/ "person.names",
/*modelFolder*/ modelFolder,
personLabelMap);
if (!ok) {
_logger.LogWarn("ANSRTRTMPOSE::TryLoadPersonDetector",
"Person detector LoadModelFromFolder FAILED -- falling back to whole-image pose",
__FILE__, __LINE__);
ANS_DBG("ANSRTRTMPOSE", "TryLoadPersonDetector: detector load FAILED");
return false;
}
m_personDetector = std::move(detector);
ANS_DBG("ANSRTRTMPOSE",
"TryLoadPersonDetector: detector loaded OK from '%s' (labels='%s')",
modelFolder.c_str(), personLabelMap.c_str());
return true;
}
catch (const std::exception& e) {
_logger.LogWarn("ANSRTRTMPOSE::TryLoadPersonDetector", e.what(), __FILE__, __LINE__);
ANS_DBG("ANSRTRTMPOSE", "TryLoadPersonDetector: EXCEPTION '%s'", e.what());
m_personDetector.reset();
return false;
}
}
} // namespace ANSCENTER
+16
View File
@@ -5,6 +5,7 @@
#include "engine.h"
#include "ANSGpuFrameRegistry.h"
#include "engine/EnginePoolManager.h"
#include "ANSRTYOLO.h" // internal person-detector front-end (top-down pipeline)
namespace ANSCENTER {
@@ -100,6 +101,21 @@ namespace ANSCENTER {
EnginePoolManager<float>::PoolKey m_poolKey;
bool m_usingSharedPool = false;
// ── Optional internal person detector (top-down pipeline) ───────
// RTMPose is a top-down single-person estimator — feeding it whole
// scenes produces noise. If the model folder bundles person.onnx +
// person.names alongside the pose model, we auto-load a YOLO person
// detector here and run pose ONCE PER DETECTED PERSON. When those
// files are missing the wrapper falls back to whole-image pose so
// that single-person test inputs (selfies, fitness clips) still work.
std::unique_ptr<ANSRTYOLO> m_personDetector;
// Look for person.onnx + person.names in `modelFolder`. Loads
// m_personDetector and returns true on success; returns false
// (without logging an error) if either file is missing — caller
// should treat that as "no detector, fall back to whole-image".
bool TryLoadPersonDetector(const std::string& modelFolder);
ANSCENTER::Options m_options;
// Fixed RTMPose model geometry (overridden from the engine's input dims