Add person model into RTMPOSE
This commit is contained in:
@@ -300,6 +300,10 @@ namespace ANSCENTER {
|
||||
|
||||
bool ANSONNXRTMPOSE::Destroy() {
|
||||
try {
|
||||
if (m_personDetector) {
|
||||
m_personDetector->Destroy();
|
||||
m_personDetector.reset();
|
||||
}
|
||||
m_ortEngine.reset();
|
||||
ANS_DBG("ANSONNXRTMPOSE", "Destroy: engine released");
|
||||
return true;
|
||||
@@ -317,6 +321,11 @@ namespace ANSCENTER {
|
||||
// ANSONNXYOLO::OptimizeModel which is also a no-op beyond folder
|
||||
// resolution.
|
||||
optimizedModelFolder = _modelFolder;
|
||||
|
||||
// Also bring up the bundled person-detector if present so its ORT
|
||||
// session is constructed (and any cached engine is materialized)
|
||||
// alongside the pose model. Mirrors the TRT counterpart's behavior.
|
||||
(void)TryLoadPersonDetector(_modelFolder);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -457,6 +466,10 @@ namespace ANSCENTER {
|
||||
WarmUpEngine();
|
||||
}
|
||||
|
||||
// Top-down pipeline: try to load the bundled person detector.
|
||||
// Falls back gracefully (whole-image pose) when the files are missing.
|
||||
(void)TryLoadPersonDetector(_modelFolder);
|
||||
|
||||
_modelLoadValid = true;
|
||||
_isInitialized = true;
|
||||
return true;
|
||||
@@ -510,6 +523,9 @@ namespace ANSCENTER {
|
||||
_modelConfig.numKPS = m_ortEngine->numKeypoints();
|
||||
WarmUpEngine();
|
||||
}
|
||||
|
||||
(void)TryLoadPersonDetector(_modelFolder);
|
||||
|
||||
_modelLoadValid = true;
|
||||
_isInitialized = true;
|
||||
return true;
|
||||
@@ -577,6 +593,9 @@ namespace ANSCENTER {
|
||||
_modelConfig.numKPS = m_ortEngine->numKeypoints();
|
||||
WarmUpEngine();
|
||||
}
|
||||
|
||||
(void)TryLoadPersonDetector(_modelFolder);
|
||||
|
||||
_modelLoadValid = true;
|
||||
_isInitialized = true;
|
||||
return true;
|
||||
@@ -623,21 +642,32 @@ namespace ANSCENTER {
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// DetectObjects — top-down: with no external bbox, treat the whole
|
||||
// input image as the person crop and run RTMPose. Higher-level
|
||||
// orchestration that already has per-person bboxes should call
|
||||
// ANSODBase::RunInference(input, vector<Rect>, cam) — that overload
|
||||
// crops each bbox and routes back here per crop.
|
||||
// DetectObjects — two pipelines:
|
||||
//
|
||||
// (a) Top-down (preferred) — m_personDetector loaded from the
|
||||
// bundled person.onnx + person.names. Detect people first,
|
||||
// then run RTMPose ONCE per person bbox. This is what RTMPose
|
||||
// is designed for; works on multi-person scenes with small /
|
||||
// distant subjects.
|
||||
//
|
||||
// (b) Standalone fallback — no detector files in the model folder.
|
||||
// Treat the whole image as a single person crop. Only sensible
|
||||
// for inputs that are already a person (selfies, single-subject
|
||||
// fitness clips, unit-test crops). Higher-level callers can
|
||||
// also use ANSODBase::RunInference(input, vector<Rect>, cam)
|
||||
// which crops each bbox and routes back here per crop.
|
||||
// ------------------------------------------------------------------
|
||||
std::vector<Object> ANSONNXRTMPOSE::DetectObjects(const cv::Mat& inputImage,
|
||||
const std::string& camera_id)
|
||||
{
|
||||
try {
|
||||
// Snapshot engine handle under brief lock; dereference outside
|
||||
// (mirrors ANSONNXYOLO::DetectObjects so concurrent inference
|
||||
// threads don't serialise on _mutex for the full ORT Run()).
|
||||
// Snapshot engine handle + detector under brief lock; dereference
|
||||
// outside (mirrors ANSONNXYOLO::DetectObjects so concurrent
|
||||
// inference threads don't serialise on _mutex for the full
|
||||
// ORT Run() chain).
|
||||
std::shared_ptr<ONNXRtmPose> engine;
|
||||
std::string firstClass;
|
||||
ANSONNXYOLO* personDetector = nullptr; // raw view; lifetime owned by m_personDetector
|
||||
std::string firstClass;
|
||||
{
|
||||
auto lk = TryLockWithTimeout("ANSONNXRTMPOSE::DetectObjects");
|
||||
if (!lk.owns_lock()) return {};
|
||||
@@ -646,10 +676,39 @@ namespace ANSCENTER {
|
||||
"ORT engine is null", __FILE__, __LINE__);
|
||||
return {};
|
||||
}
|
||||
engine = m_ortEngine; // ref-count++
|
||||
engine = m_ortEngine; // ref-count++
|
||||
personDetector = m_personDetector.get();
|
||||
if (!_classes.empty()) firstClass = _classes.front();
|
||||
}
|
||||
|
||||
// --- Top-down path ---
|
||||
if (personDetector) {
|
||||
std::vector<Object> persons = personDetector->RunInference(inputImage, camera_id);
|
||||
if (persons.empty()) {
|
||||
ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: detector found 0 persons cam='%s'",
|
||||
camera_id.c_str());
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<Object> out;
|
||||
out.reserve(persons.size());
|
||||
for (const auto& person : persons) {
|
||||
if (person.box.width <= 0 || person.box.height <= 0) continue;
|
||||
Object pose = engine->estimate(inputImage, person.box);
|
||||
if (pose.kps.empty()) continue;
|
||||
pose.cameraId = camera_id;
|
||||
pose.trackId = person.trackId;
|
||||
pose.confidence = person.confidence;
|
||||
pose.className = !person.className.empty() ? person.className
|
||||
: (firstClass.empty() ? std::string("person") : firstClass);
|
||||
out.push_back(std::move(pose));
|
||||
}
|
||||
ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: %zu persons -> %zu poses cam='%s'",
|
||||
persons.size(), out.size(), camera_id.c_str());
|
||||
return out;
|
||||
}
|
||||
|
||||
// --- Standalone fallback path ---
|
||||
const cv::Rect wholeImage(0, 0, inputImage.cols, inputImage.rows);
|
||||
Object obj = engine->estimate(inputImage, wholeImage);
|
||||
if (obj.kps.empty()) return {};
|
||||
@@ -659,7 +718,7 @@ namespace ANSCENTER {
|
||||
|
||||
std::vector<Object> out;
|
||||
out.push_back(std::move(obj));
|
||||
ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: kps=%d cam='%s'",
|
||||
ANS_DBG("ANSONNXRTMPOSE", "DetectObjects (standalone): kps=%d cam='%s'",
|
||||
engine->numKeypoints(), camera_id.c_str());
|
||||
return out;
|
||||
}
|
||||
@@ -670,4 +729,78 @@ namespace ANSCENTER {
|
||||
}
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// TryLoadPersonDetector
|
||||
//
|
||||
// Look for person.onnx + person.names alongside the pose model. If
|
||||
// both exist, instantiate an ANSONNXYOLO front-end and load it via
|
||||
// LoadModelFromFolder (which creates the ORT session). On success
|
||||
// m_personDetector is non-null and DetectObjects switches to the
|
||||
// proper top-down pipeline.
|
||||
//
|
||||
// Missing files are NOT an error — the wrapper is designed to work
|
||||
// standalone for single-person test inputs. We log a debug line so
|
||||
// it's clear which path is active and return false.
|
||||
// ====================================================================
|
||||
bool ANSONNXRTMPOSE::TryLoadPersonDetector(const std::string& modelFolder) {
|
||||
const std::string personOnnx = CreateFilePath(modelFolder, "person.onnx");
|
||||
const std::string personNames = CreateFilePath(modelFolder, "person.names");
|
||||
if (!FileExist(personOnnx) || !FileExist(personNames)) {
|
||||
ANS_DBG("ANSONNXRTMPOSE",
|
||||
"TryLoadPersonDetector: person.onnx/person.names not in '%s' "
|
||||
"-- standalone mode (whole-image pose)", modelFolder.c_str());
|
||||
return false;
|
||||
}
|
||||
if (m_personDetector) {
|
||||
ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: already loaded, skipping");
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
auto detector = std::make_unique<ANSONNXYOLO>();
|
||||
detector->SetLoadEngineOnCreation(this->_loadEngineOnCreation);
|
||||
|
||||
ModelConfig personCfg{};
|
||||
personCfg.detectionType = DetectionType::DETECTION;
|
||||
personCfg.modelType = ModelType::ONNXYOLO;
|
||||
personCfg.precisionType = PrecisionType::FP32;
|
||||
personCfg.inpHeight = 640;
|
||||
personCfg.inpWidth = 640;
|
||||
personCfg.detectionScoreThreshold = 0.5f;
|
||||
personCfg.modelConfThreshold = 0.5f;
|
||||
personCfg.modelMNSThreshold = 0.45f;
|
||||
personCfg.numKPS = 0;
|
||||
personCfg.gpuOptBatchSize = 1;
|
||||
personCfg.gpuMaxBatchSize = 1;
|
||||
|
||||
std::string personLabelMap;
|
||||
const bool ok = detector->LoadModelFromFolder(
|
||||
/*licenseKey*/ _licenseKey,
|
||||
personCfg,
|
||||
/*modelName*/ "person",
|
||||
/*className*/ "person.names",
|
||||
/*modelFolder*/ modelFolder,
|
||||
personLabelMap);
|
||||
if (!ok) {
|
||||
_logger.LogWarn("ANSONNXRTMPOSE::TryLoadPersonDetector",
|
||||
"Person detector LoadModelFromFolder FAILED -- falling back to whole-image pose",
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: detector load FAILED");
|
||||
return false;
|
||||
}
|
||||
|
||||
m_personDetector = std::move(detector);
|
||||
ANS_DBG("ANSONNXRTMPOSE",
|
||||
"TryLoadPersonDetector: detector loaded OK from '%s' (labels='%s')",
|
||||
modelFolder.c_str(), personLabelMap.c_str());
|
||||
return true;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
_logger.LogWarn("ANSONNXRTMPOSE::TryLoadPersonDetector", e.what(), __FILE__, __LINE__);
|
||||
ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: EXCEPTION '%s'", e.what());
|
||||
m_personDetector.reset();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace ANSCENTER
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#pragma once
|
||||
#include "ANSEngineCommon.h"
|
||||
#include "ONNXEngine.h"
|
||||
#include "ANSONNXYOLO.h" // internal person-detector front-end (top-down pipeline)
|
||||
|
||||
namespace ANSCENTER {
|
||||
|
||||
@@ -144,6 +145,21 @@ namespace ANSCENTER {
|
||||
// member here on another thread.
|
||||
std::shared_ptr<ONNXRtmPose> m_ortEngine;
|
||||
|
||||
// ── Optional internal person detector (top-down pipeline) ───────
|
||||
// RTMPose is a top-down single-person estimator. If the model
|
||||
// folder bundles person.onnx + person.names alongside the pose
|
||||
// model, we auto-load an ANSONNXYOLO person detector and run pose
|
||||
// ONCE PER DETECTED PERSON. When those files are missing the
|
||||
// wrapper falls back to whole-image pose so single-person test
|
||||
// inputs (selfies, fitness clips) still work.
|
||||
std::unique_ptr<ANSONNXYOLO> m_personDetector;
|
||||
|
||||
// Look for person.onnx + person.names in `modelFolder`. Loads
|
||||
// m_personDetector and returns true on success; returns false
|
||||
// (without logging an error) if either file is missing — caller
|
||||
// should treat that as "no detector, fall back to whole-image".
|
||||
bool TryLoadPersonDetector(const std::string& modelFolder);
|
||||
|
||||
bool InitOrtEngine();
|
||||
bool InitOrtEngine(ANSCENTER::EngineType engineType);
|
||||
void WarmUpEngine();
|
||||
|
||||
@@ -46,6 +46,12 @@ namespace ANSCENTER {
|
||||
return false;
|
||||
}
|
||||
ANS_DBG("ANSRTRTMPOSE", "OptimizeModel: build OK -> '%s'", optimizedModelFolder.c_str());
|
||||
|
||||
// Also build the bundled person-detector engine if present.
|
||||
// TryLoadPersonDetector triggers ANSRTYOLO::LoadModelFromFolder
|
||||
// which performs the engine build (= optimization) and caches
|
||||
// it to disk; the next Initialize/LoadModel will hit the cache.
|
||||
(void)TryLoadPersonDetector(_modelFolder);
|
||||
return true;
|
||||
}
|
||||
catch (std::exception& e) {
|
||||
@@ -138,6 +144,11 @@ namespace ANSCENTER {
|
||||
ANS_DBG("ANSRTRTMPOSE", "LoadModel: engine ready %dx%d kps=%d sharedPool=%d",
|
||||
RTM_INPUT_W, RTM_INPUT_H, _modelConfig.numKPS, m_usingSharedPool ? 1 : 0);
|
||||
}
|
||||
|
||||
// Top-down pipeline: try to load the bundled person detector.
|
||||
// Falls back gracefully (whole-image pose) when the files are missing.
|
||||
(void)TryLoadPersonDetector(_modelFolder);
|
||||
|
||||
_modelLoadValid = true;
|
||||
_isInitialized = true;
|
||||
return true;
|
||||
@@ -242,6 +253,9 @@ namespace ANSCENTER {
|
||||
ANS_DBG("ANSRTRTMPOSE", "LoadModelFromFolder: engine ready %dx%d kps=%d sharedPool=%d",
|
||||
RTM_INPUT_W, RTM_INPUT_H, _modelConfig.numKPS, m_usingSharedPool ? 1 : 0);
|
||||
}
|
||||
|
||||
(void)TryLoadPersonDetector(_modelFolder);
|
||||
|
||||
_modelLoadValid = true;
|
||||
_isInitialized = true;
|
||||
return true;
|
||||
@@ -344,6 +358,9 @@ namespace ANSCENTER {
|
||||
ANS_DBG("ANSRTRTMPOSE", "Initialize: engine ready %dx%d kps=%d sharedPool=%d",
|
||||
RTM_INPUT_W, RTM_INPUT_H, _modelConfig.numKPS, m_usingSharedPool ? 1 : 0);
|
||||
}
|
||||
|
||||
(void)TryLoadPersonDetector(_modelFolder);
|
||||
|
||||
_modelLoadValid = true;
|
||||
_isInitialized = true;
|
||||
return true;
|
||||
@@ -370,11 +387,56 @@ namespace ANSCENTER {
|
||||
return {};
|
||||
}
|
||||
try {
|
||||
// Top-down pose estimator: with no external bbox, treat the whole
|
||||
// image as a single person crop. Higher-level orchestration that
|
||||
// wants per-person pose should call ANSODBase::RunInference(input,
|
||||
// bboxes, cam) instead — that overload crops each bbox and routes
|
||||
// back through us per crop.
|
||||
// Two pipelines:
|
||||
//
|
||||
// (a) Top-down (preferred) — m_personDetector loaded from
|
||||
// the bundled person.onnx + person.names. Detect people
|
||||
// first, then run RTMPose ONCE per person bbox. This is
|
||||
// what RTMPose is designed for; works on multi-person
|
||||
// scenes with small / distant subjects.
|
||||
//
|
||||
// (b) Standalone fallback — no detector files in the model
|
||||
// folder. Treat the whole image as a single person crop.
|
||||
// Only sensible for inputs that are already a person
|
||||
// (selfies, single-subject fitness clips, unit-test
|
||||
// crops). Higher-level callers can also use the
|
||||
// ANSODBase::RunInference(input, vector<Rect>, cam)
|
||||
// overload which crops each bbox and routes per-crop
|
||||
// through our Preprocess.
|
||||
if (m_personDetector) {
|
||||
std::vector<Object> persons = m_personDetector->RunInference(inputImgBGR, camera_id);
|
||||
if (persons.empty()) {
|
||||
ANS_DBG("ANSRTRTMPOSE", "RunInference: detector found 0 persons cam='%s'",
|
||||
camera_id.c_str());
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<Object> results;
|
||||
results.reserve(persons.size());
|
||||
for (const auto& person : persons) {
|
||||
// person.onnx is a single-class model — accept anything
|
||||
// the detector returned, but defend against degenerate
|
||||
// boxes (zero/negative area, off-image).
|
||||
if (person.box.width <= 0 || person.box.height <= 0) continue;
|
||||
|
||||
auto poseResults = InferenceOnBox(inputImgBGR, person.box, camera_id);
|
||||
for (auto& obj : poseResults) {
|
||||
// Carry the detector's track-id and confidence so
|
||||
// downstream tracking/visualization stays coherent
|
||||
// across frames. RTMPose itself has no class /
|
||||
// tracking concept of its own.
|
||||
obj.trackId = person.trackId;
|
||||
obj.confidence = person.confidence;
|
||||
if (!person.className.empty()) obj.className = person.className;
|
||||
results.push_back(std::move(obj));
|
||||
}
|
||||
}
|
||||
ANS_DBG("ANSRTRTMPOSE", "RunInference: %zu persons -> %zu poses cam='%s'",
|
||||
persons.size(), results.size(), camera_id.c_str());
|
||||
return results;
|
||||
}
|
||||
|
||||
// Fallback — single-person whole-image pose
|
||||
const cv::Rect wholeImage(0, 0, inputImgBGR.cols, inputImgBGR.rows);
|
||||
return InferenceOnBox(inputImgBGR, wholeImage, camera_id);
|
||||
}
|
||||
@@ -394,6 +456,10 @@ namespace ANSCENTER {
|
||||
|
||||
bool ANSRTRTMPOSE::Destroy() {
|
||||
try {
|
||||
if (m_personDetector) {
|
||||
m_personDetector->Destroy();
|
||||
m_personDetector.reset();
|
||||
}
|
||||
if (m_usingSharedPool) {
|
||||
EnginePoolManager<float>::instance().release(m_poolKey);
|
||||
m_trtEngine.reset();
|
||||
@@ -699,4 +765,83 @@ namespace ANSCENTER {
|
||||
cv::Size(RTM_INPUT_W, RTM_INPUT_H), cv::INTER_LINEAR);
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// TryLoadPersonDetector
|
||||
//
|
||||
// Look for person.onnx + person.names alongside the pose model. If
|
||||
// both exist, instantiate an ANSRTYOLO front-end and load it via
|
||||
// LoadModelFromFolder (which builds / loads the TRT engine from
|
||||
// cache). On success m_personDetector is non-null and RunInference
|
||||
// switches to the proper top-down pipeline.
|
||||
//
|
||||
// Missing files are NOT an error — the wrapper is designed to work
|
||||
// standalone for single-person test inputs. We log a debug line so
|
||||
// it's clear which path is active and return false.
|
||||
// ====================================================================
|
||||
bool ANSRTRTMPOSE::TryLoadPersonDetector(const std::string& modelFolder) {
|
||||
const std::string personOnnx = CreateFilePath(modelFolder, "person.onnx");
|
||||
const std::string personNames = CreateFilePath(modelFolder, "person.names");
|
||||
if (!FileExist(personOnnx) || !FileExist(personNames)) {
|
||||
ANS_DBG("ANSRTRTMPOSE",
|
||||
"TryLoadPersonDetector: person.onnx/person.names not in '%s' "
|
||||
"-- standalone mode (whole-image pose)", modelFolder.c_str());
|
||||
return false;
|
||||
}
|
||||
if (m_personDetector) {
|
||||
ANS_DBG("ANSRTRTMPOSE", "TryLoadPersonDetector: already loaded, skipping");
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
auto detector = std::make_unique<ANSRTYOLO>();
|
||||
detector->SetLoadEngineOnCreation(this->_loadEngineOnCreation);
|
||||
// SetMaxSlotsPerGpu is declared private in ANSRTYOLO (it's a
|
||||
// private override of ANSODBase's public virtual). Call it via
|
||||
// the base-class interface where access is public.
|
||||
static_cast<ANSODBase*>(detector.get())->SetMaxSlotsPerGpu(m_maxSlotsPerGpu);
|
||||
|
||||
ModelConfig personCfg{};
|
||||
personCfg.detectionType = DetectionType::DETECTION;
|
||||
personCfg.modelType = ModelType::RTYOLO;
|
||||
personCfg.precisionType = PrecisionType::FP16;
|
||||
personCfg.inpHeight = 640;
|
||||
personCfg.inpWidth = 640;
|
||||
personCfg.detectionScoreThreshold = 0.5f;
|
||||
personCfg.modelConfThreshold = 0.5f;
|
||||
personCfg.modelMNSThreshold = 0.45f;
|
||||
personCfg.numKPS = 0;
|
||||
personCfg.gpuOptBatchSize = 1;
|
||||
personCfg.gpuMaxBatchSize = 1;
|
||||
personCfg.gpuDeviceIndex = m_options.deviceIndex;
|
||||
|
||||
std::string personLabelMap;
|
||||
const bool ok = detector->LoadModelFromFolder(
|
||||
/*licenseKey*/ _licenseKey,
|
||||
personCfg,
|
||||
/*modelName*/ "person",
|
||||
/*className*/ "person.names",
|
||||
/*modelFolder*/ modelFolder,
|
||||
personLabelMap);
|
||||
if (!ok) {
|
||||
_logger.LogWarn("ANSRTRTMPOSE::TryLoadPersonDetector",
|
||||
"Person detector LoadModelFromFolder FAILED -- falling back to whole-image pose",
|
||||
__FILE__, __LINE__);
|
||||
ANS_DBG("ANSRTRTMPOSE", "TryLoadPersonDetector: detector load FAILED");
|
||||
return false;
|
||||
}
|
||||
|
||||
m_personDetector = std::move(detector);
|
||||
ANS_DBG("ANSRTRTMPOSE",
|
||||
"TryLoadPersonDetector: detector loaded OK from '%s' (labels='%s')",
|
||||
modelFolder.c_str(), personLabelMap.c_str());
|
||||
return true;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
_logger.LogWarn("ANSRTRTMPOSE::TryLoadPersonDetector", e.what(), __FILE__, __LINE__);
|
||||
ANS_DBG("ANSRTRTMPOSE", "TryLoadPersonDetector: EXCEPTION '%s'", e.what());
|
||||
m_personDetector.reset();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace ANSCENTER
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "engine.h"
|
||||
#include "ANSGpuFrameRegistry.h"
|
||||
#include "engine/EnginePoolManager.h"
|
||||
#include "ANSRTYOLO.h" // internal person-detector front-end (top-down pipeline)
|
||||
|
||||
namespace ANSCENTER {
|
||||
|
||||
@@ -100,6 +101,21 @@ namespace ANSCENTER {
|
||||
EnginePoolManager<float>::PoolKey m_poolKey;
|
||||
bool m_usingSharedPool = false;
|
||||
|
||||
// ── Optional internal person detector (top-down pipeline) ───────
|
||||
// RTMPose is a top-down single-person estimator — feeding it whole
|
||||
// scenes produces noise. If the model folder bundles person.onnx +
|
||||
// person.names alongside the pose model, we auto-load a YOLO person
|
||||
// detector here and run pose ONCE PER DETECTED PERSON. When those
|
||||
// files are missing the wrapper falls back to whole-image pose so
|
||||
// that single-person test inputs (selfies, fitness clips) still work.
|
||||
std::unique_ptr<ANSRTYOLO> m_personDetector;
|
||||
|
||||
// Look for person.onnx + person.names in `modelFolder`. Loads
|
||||
// m_personDetector and returns true on success; returns false
|
||||
// (without logging an error) if either file is missing — caller
|
||||
// should treat that as "no detector, fall back to whole-image".
|
||||
bool TryLoadPersonDetector(const std::string& modelFolder);
|
||||
|
||||
ANSCENTER::Options m_options;
|
||||
|
||||
// Fixed RTMPose model geometry (overridden from the engine's input dims
|
||||
|
||||
@@ -971,16 +971,19 @@ int VideoDetectorEngine(std::string modelFilePath, std::string videoFilePath, in
|
||||
printf("Inference Time = %lld ms\n", static_cast<long long int>(elapsed.count()));
|
||||
|
||||
if (!detectionResult.empty()) {
|
||||
// COCO pose skeleton connections
|
||||
const std::vector<std::pair<int, int>> POSE_SKELETON = {
|
||||
// ── COCO 17-keypoint skeleton (Ultralytics YOLO Pose format) ──
|
||||
// Indices: 0=nose, 1=Leye, 2=Reye, 3=Lear, 4=Rear,
|
||||
// 5=Lshoulder, 6=Rshoulder, 7=Lelbow, 8=Relbow,
|
||||
// 9=Lwrist, 10=Rwrist, 11=Lhip, 12=Rhip,
|
||||
// 13=Lknee, 14=Rknee, 15=Lankle, 16=Rankle
|
||||
const std::vector<std::pair<int, int>> COCO17_SKELETON = {
|
||||
{0,1}, {0,2}, {1,3}, {2,4}, // Face
|
||||
{3,5}, {4,6}, // Head-to-shoulder
|
||||
{5,7}, {7,9}, {6,8}, {8,10}, // Arms
|
||||
{5,6}, {5,11}, {6,12}, {11,12}, // Body
|
||||
{11,13}, {13,15}, {12,14}, {14,16} // Legs
|
||||
};
|
||||
// Colors for different limb groups
|
||||
const std::vector<cv::Scalar> LIMB_COLORS = {
|
||||
const std::vector<cv::Scalar> COCO17_COLORS = {
|
||||
{255,0,0}, {255,0,0}, {255,0,0}, {255,0,0}, // Face - blue
|
||||
{0,255,255}, {0,255,255}, // Head-shoulder - yellow
|
||||
{0,255,0}, {0,255,0}, {0,165,255}, {0,165,255}, // Arms - green/orange
|
||||
@@ -988,6 +991,68 @@ int VideoDetectorEngine(std::string modelFilePath, std::string videoFilePath, in
|
||||
{0,255,0}, {0,255,0}, {0,165,255}, {0,165,255} // Legs - green/orange
|
||||
};
|
||||
|
||||
// ── Halpe 26-keypoint skeleton (MMPose body_2d_keypoint, used by RTMPose) ──
|
||||
// Layout: 0..16 = COCO 17 (same as above);
|
||||
// 17=Head, 18=Neck, 19=Hip(mid),
|
||||
// 20=LBigToe, 21=RBigToe,
|
||||
// 22=LSmallToe, 23=RSmallToe,
|
||||
// 24=LHeel, 25=RHeel
|
||||
const std::vector<std::pair<int, int>> HALPE26_SKELETON = {
|
||||
{0,1}, {0,2}, {1,3}, {2,4}, // face
|
||||
{17,18}, // head -> neck
|
||||
{5,18}, {6,18}, // shoulders -> neck
|
||||
{18,19}, // neck -> mid hip
|
||||
{19,11}, {19,12}, // mid hip -> L/R hip
|
||||
{5,7}, {7,9}, // left arm
|
||||
{6,8}, {8,10}, // right arm
|
||||
{11,13}, {13,15}, // left leg
|
||||
{12,14}, {14,16}, // right leg
|
||||
{15,20}, {15,22}, {15,24}, // left foot (big-toe / small-toe / heel)
|
||||
{16,21}, {16,23}, {16,25} // right foot
|
||||
};
|
||||
const std::vector<cv::Scalar> HALPE26_COLORS = {
|
||||
{255,0,0}, {255,0,0}, {255,0,0}, {255,0,0}, // face - blue
|
||||
{0,255,255}, // head-neck - yellow
|
||||
{0,255,255}, {0,255,255}, // shoulders-neck - yellow
|
||||
{255,255,0}, // neck-midhip - cyan
|
||||
{255,255,0}, {255,255,0}, // midhip-hips - cyan
|
||||
{0,255,0}, {0,255,0}, // left arm - green
|
||||
{0,165,255}, {0,165,255}, // right arm - orange
|
||||
{0,255,0}, {0,255,0}, // left leg - green
|
||||
{0,165,255}, {0,165,255}, // right leg - orange
|
||||
{200,100,255}, {200,100,255}, {200,100,255}, // left foot - light purple
|
||||
{255,100,100}, {255,100,100}, {255,100,100} // right foot - light pink
|
||||
};
|
||||
|
||||
// ── OpenPose 18-keypoint skeleton (OpenVINO HumanPose, used by ANSPOSE) ──
|
||||
// DIFFERENT index layout from COCO-17:
|
||||
// 0=Nose, 1=Neck, 2=RShoulder, 3=RElbow, 4=RWrist,
|
||||
// 5=LShoulder, 6=LElbow, 7=LWrist,
|
||||
// 8=RHip, 9=RKnee, 10=RAnkle,
|
||||
// 11=LHip, 12=LKnee, 13=LAnkle,
|
||||
// 14=REye, 15=LEye, 16=REar, 17=LEar
|
||||
// Note: ANSPOSE writes kps as stride-2 (x,y only — no per-kp score),
|
||||
// so the drawer detects this layout by buffer size and skips
|
||||
// the confidence threshold for it.
|
||||
const std::vector<std::pair<int, int>> OPENPOSE18_SKELETON = {
|
||||
{1,2}, {2,3}, {3,4}, // right arm (neck->shoulder->elbow->wrist)
|
||||
{1,5}, {5,6}, {6,7}, // left arm
|
||||
{1,8}, {8,9}, {9,10}, // right leg (neck->hip->knee->ankle)
|
||||
{1,11}, {11,12}, {12,13}, // left leg
|
||||
{1,0}, // neck->nose
|
||||
{0,14}, {14,16}, // right eye->ear
|
||||
{0,15}, {15,17} // left eye->ear
|
||||
};
|
||||
const std::vector<cv::Scalar> OPENPOSE18_COLORS = {
|
||||
{0,165,255}, {0,165,255}, {0,165,255}, // right arm - orange
|
||||
{0,255,0}, {0,255,0}, {0,255,0}, // left arm - green
|
||||
{0,165,255}, {0,165,255}, {0,165,255}, // right leg - orange
|
||||
{0,255,0}, {0,255,0}, {0,255,0}, // left leg - green
|
||||
{255,255,0}, // neck-nose - cyan
|
||||
{255,0,0}, {255,0,0}, // right face - blue
|
||||
{255,0,0}, {255,0,0} // left face - blue
|
||||
};
|
||||
|
||||
pt.clear();
|
||||
std::stringstream ss;
|
||||
ss.clear();
|
||||
@@ -1103,29 +1168,88 @@ int VideoDetectorEngine(std::string modelFilePath, std::string videoFilePath, in
|
||||
}
|
||||
}
|
||||
|
||||
// Draw keypoints if available (format: x1;y1;score1;x2;y2;score2;... stride=3)
|
||||
// Draw keypoints if available. Three on-the-wire formats are
|
||||
// auto-detected from the buffer length so the same drawer
|
||||
// handles all pose models without changing the engines:
|
||||
//
|
||||
// 51 floats → 17 kps stride-3 → YOLO Pose (COCO-17)
|
||||
// 78 floats → 26 kps stride-3 → RTMPose (Halpe-26)
|
||||
// 36 floats → 18 kps stride-2 → OpenPose/ANSPOSE
|
||||
//
|
||||
// For stride-3 formats kps[k*3+2] is a per-keypoint confidence
|
||||
// and we threshold on it. ANSPOSE doesn't emit confidences
|
||||
// (kps are stride-2), so we use kp.x>0 && kp.y>0 as the
|
||||
// visibility proxy (ANSPOSE writes 0 / negatives for missing
|
||||
// keypoints — see ANSPOSE.cpp:292).
|
||||
if (!kps_str.empty()) {
|
||||
std::vector<float> kps = ANSCENTER::ANSUtilityHelper::StringToKeypoints(kps_str);
|
||||
int numKeypoints = (int)kps.size() / 3; // stride of 3: x, y, confidence
|
||||
const size_t n = kps.size();
|
||||
|
||||
// Collect keypoint positions and confidence
|
||||
const std::vector<std::pair<int, int>>* skeleton = nullptr;
|
||||
const std::vector<cv::Scalar>* colors = nullptr;
|
||||
int stride = 3;
|
||||
int numKeypoints = 0;
|
||||
bool hasPerKpScore = true;
|
||||
|
||||
if (n == 51) { // YOLO Pose: 17 kps × 3
|
||||
stride = 3;
|
||||
numKeypoints = 17;
|
||||
skeleton = &COCO17_SKELETON;
|
||||
colors = &COCO17_COLORS;
|
||||
hasPerKpScore = true;
|
||||
}
|
||||
else if (n == 78) { // RTMPose: 26 kps × 3
|
||||
stride = 3;
|
||||
numKeypoints = 26;
|
||||
skeleton = &HALPE26_SKELETON;
|
||||
colors = &HALPE26_COLORS;
|
||||
hasPerKpScore = true;
|
||||
}
|
||||
else if (n == 36) { // OpenPose/ANSPOSE: 18 kps × 2
|
||||
stride = 2;
|
||||
numKeypoints = 18;
|
||||
skeleton = &OPENPOSE18_SKELETON;
|
||||
colors = &OPENPOSE18_COLORS;
|
||||
hasPerKpScore = false;
|
||||
}
|
||||
else if (n % 3 == 0) { // Unknown stride-3 — draw dots only
|
||||
stride = 3;
|
||||
numKeypoints = static_cast<int>(n) / 3;
|
||||
hasPerKpScore = true;
|
||||
}
|
||||
else if (n % 2 == 0) { // Unknown stride-2 — draw dots only
|
||||
stride = 2;
|
||||
numKeypoints = static_cast<int>(n) / 2;
|
||||
hasPerKpScore = false;
|
||||
}
|
||||
|
||||
// Collect keypoint positions and (optional) confidence.
|
||||
// For stride-2 we treat any kp with x>0 && y>0 as visible.
|
||||
std::vector<cv::Point2f> keypointPts(numKeypoints);
|
||||
std::vector<float> keypointConf(numKeypoints, 0.f);
|
||||
std::vector<float> keypointConf(numKeypoints, 1.f);
|
||||
for (int k = 0; k < numKeypoints; k++) {
|
||||
keypointPts[k] = cv::Point2f(kps[k * 3], kps[k * 3 + 1]);
|
||||
keypointConf[k] = kps[k * 3 + 2];
|
||||
const float kx = kps[k * stride];
|
||||
const float ky = kps[k * stride + 1];
|
||||
keypointPts[k] = cv::Point2f(kx, ky);
|
||||
if (hasPerKpScore) {
|
||||
keypointConf[k] = kps[k * stride + 2];
|
||||
} else {
|
||||
keypointConf[k] = (kx > 0.f && ky > 0.f) ? 1.f : 0.f;
|
||||
}
|
||||
}
|
||||
|
||||
const float kpConfThreshold = 0.3f;
|
||||
|
||||
// Draw skeleton lines
|
||||
for (size_t s = 0; s < POSE_SKELETON.size(); s++) {
|
||||
int idx1 = POSE_SKELETON[s].first;
|
||||
int idx2 = POSE_SKELETON[s].second;
|
||||
if (idx1 < numKeypoints && idx2 < numKeypoints &&
|
||||
keypointConf[idx1] > kpConfThreshold && keypointConf[idx2] > kpConfThreshold) {
|
||||
cv::Scalar color = (s < LIMB_COLORS.size()) ? LIMB_COLORS[s] : cv::Scalar(200, 200, 200);
|
||||
cv::line(frame, keypointPts[idx1], keypointPts[idx2], color, 2, cv::LINE_AA);
|
||||
// Draw skeleton lines (skipped for unknown layouts).
|
||||
if (skeleton) {
|
||||
for (size_t s = 0; s < skeleton->size(); s++) {
|
||||
int idx1 = (*skeleton)[s].first;
|
||||
int idx2 = (*skeleton)[s].second;
|
||||
if (idx1 < numKeypoints && idx2 < numKeypoints &&
|
||||
keypointConf[idx1] > kpConfThreshold && keypointConf[idx2] > kpConfThreshold) {
|
||||
cv::Scalar color = (s < colors->size()) ? (*colors)[s] : cv::Scalar(200, 200, 200);
|
||||
cv::line(frame, keypointPts[idx1], keypointPts[idx2], color, 2, cv::LINE_AA);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1488,7 +1612,6 @@ int YOLO26POSEYolo11Test() {
|
||||
int modelType = 31; // ONNX YOLO (30) RT YOLO (31)
|
||||
std::string onnxclmodelFilePath = "E:\\Programs\\DemoAssets\\ANSAIModels\\PoseEstimationModels\\ANSPOSEN.zip";
|
||||
std::string videoFile = "E:\\Programs\\DemoAssets\\Videos\\train.mp4";
|
||||
|
||||
VideoDetectorEngine(onnxclmodelFilePath, videoFile, modelType);// 22 is RT, 21 is onnx
|
||||
return 0;
|
||||
}
|
||||
@@ -1872,14 +1995,23 @@ int FallDetectorTest() {
|
||||
VideoDetectorEngine(modelFilePath, videoFile, modelType);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int RTMPOSETest() {
|
||||
int modelType = 33; // ONNX YOLO (30) RT YOLO (31)
|
||||
std::string onnxclmodelFilePath = "E:\\Programs\\DemoAssets\\ANSAIModels\\PoseEstimationModels\\ANSRTMPOSE.zip";
|
||||
std::string videoFile = "E:\\Programs\\DemoAssets\\Videos\\train.mp4";
|
||||
VideoDetectorEngine(onnxclmodelFilePath, videoFile, modelType);// 22 is RT, 21 is onnx
|
||||
return 0;
|
||||
}
|
||||
int main()
|
||||
{
|
||||
|
||||
FallDetectorTest();
|
||||
//FallDetectorTest();
|
||||
//ALPROD_Test();
|
||||
//FireNSmokeTest();
|
||||
//ElasticGPUTest();
|
||||
|
||||
RTMPOSETest();
|
||||
//YOLO26SEGYolo11Test();
|
||||
//YOLO26POSEYolo11Test();
|
||||
//YOLO26CLYolo11Test();
|
||||
|
||||
Reference in New Issue
Block a user