diff --git a/modules/ANSODEngine/ANSONNXRTMPOSE.cpp b/modules/ANSODEngine/ANSONNXRTMPOSE.cpp index e69c71a..e5548d5 100644 --- a/modules/ANSODEngine/ANSONNXRTMPOSE.cpp +++ b/modules/ANSODEngine/ANSONNXRTMPOSE.cpp @@ -300,6 +300,10 @@ namespace ANSCENTER { bool ANSONNXRTMPOSE::Destroy() { try { + if (m_personDetector) { + m_personDetector->Destroy(); + m_personDetector.reset(); + } m_ortEngine.reset(); ANS_DBG("ANSONNXRTMPOSE", "Destroy: engine released"); return true; @@ -317,6 +321,11 @@ namespace ANSCENTER { // ANSONNXYOLO::OptimizeModel which is also a no-op beyond folder // resolution. optimizedModelFolder = _modelFolder; + + // Also bring up the bundled person-detector if present so its ORT + // session is constructed (and any cached engine is materialized) + // alongside the pose model. Mirrors the TRT counterpart's behavior. + (void)TryLoadPersonDetector(_modelFolder); return true; } @@ -457,6 +466,10 @@ namespace ANSCENTER { WarmUpEngine(); } + // Top-down pipeline: try to load the bundled person detector. + // Falls back gracefully (whole-image pose) when the files are missing. + (void)TryLoadPersonDetector(_modelFolder); + _modelLoadValid = true; _isInitialized = true; return true; @@ -510,6 +523,9 @@ namespace ANSCENTER { _modelConfig.numKPS = m_ortEngine->numKeypoints(); WarmUpEngine(); } + + (void)TryLoadPersonDetector(_modelFolder); + _modelLoadValid = true; _isInitialized = true; return true; @@ -577,6 +593,9 @@ namespace ANSCENTER { _modelConfig.numKPS = m_ortEngine->numKeypoints(); WarmUpEngine(); } + + (void)TryLoadPersonDetector(_modelFolder); + _modelLoadValid = true; _isInitialized = true; return true; @@ -623,21 +642,32 @@ namespace ANSCENTER { } // ------------------------------------------------------------------ - // DetectObjects — top-down: with no external bbox, treat the whole - // input image as the person crop and run RTMPose. Higher-level - // orchestration that already has per-person bboxes should call - // ANSODBase::RunInference(input, vector, cam) — that overload - // crops each bbox and routes back here per crop. + // DetectObjects — two pipelines: + // + // (a) Top-down (preferred) — m_personDetector loaded from the + // bundled person.onnx + person.names. Detect people first, + // then run RTMPose ONCE per person bbox. This is what RTMPose + // is designed for; works on multi-person scenes with small / + // distant subjects. + // + // (b) Standalone fallback — no detector files in the model folder. + // Treat the whole image as a single person crop. Only sensible + // for inputs that are already a person (selfies, single-subject + // fitness clips, unit-test crops). Higher-level callers can + // also use ANSODBase::RunInference(input, vector, cam) + // which crops each bbox and routes back here per crop. // ------------------------------------------------------------------ std::vector ANSONNXRTMPOSE::DetectObjects(const cv::Mat& inputImage, const std::string& camera_id) { try { - // Snapshot engine handle under brief lock; dereference outside - // (mirrors ANSONNXYOLO::DetectObjects so concurrent inference - // threads don't serialise on _mutex for the full ORT Run()). + // Snapshot engine handle + detector under brief lock; dereference + // outside (mirrors ANSONNXYOLO::DetectObjects so concurrent + // inference threads don't serialise on _mutex for the full + // ORT Run() chain). std::shared_ptr engine; - std::string firstClass; + ANSONNXYOLO* personDetector = nullptr; // raw view; lifetime owned by m_personDetector + std::string firstClass; { auto lk = TryLockWithTimeout("ANSONNXRTMPOSE::DetectObjects"); if (!lk.owns_lock()) return {}; @@ -646,10 +676,39 @@ namespace ANSCENTER { "ORT engine is null", __FILE__, __LINE__); return {}; } - engine = m_ortEngine; // ref-count++ + engine = m_ortEngine; // ref-count++ + personDetector = m_personDetector.get(); if (!_classes.empty()) firstClass = _classes.front(); } + // --- Top-down path --- + if (personDetector) { + std::vector persons = personDetector->RunInference(inputImage, camera_id); + if (persons.empty()) { + ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: detector found 0 persons cam='%s'", + camera_id.c_str()); + return {}; + } + + std::vector out; + out.reserve(persons.size()); + for (const auto& person : persons) { + if (person.box.width <= 0 || person.box.height <= 0) continue; + Object pose = engine->estimate(inputImage, person.box); + if (pose.kps.empty()) continue; + pose.cameraId = camera_id; + pose.trackId = person.trackId; + pose.confidence = person.confidence; + pose.className = !person.className.empty() ? person.className + : (firstClass.empty() ? std::string("person") : firstClass); + out.push_back(std::move(pose)); + } + ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: %zu persons -> %zu poses cam='%s'", + persons.size(), out.size(), camera_id.c_str()); + return out; + } + + // --- Standalone fallback path --- const cv::Rect wholeImage(0, 0, inputImage.cols, inputImage.rows); Object obj = engine->estimate(inputImage, wholeImage); if (obj.kps.empty()) return {}; @@ -659,7 +718,7 @@ namespace ANSCENTER { std::vector out; out.push_back(std::move(obj)); - ANS_DBG("ANSONNXRTMPOSE", "DetectObjects: kps=%d cam='%s'", + ANS_DBG("ANSONNXRTMPOSE", "DetectObjects (standalone): kps=%d cam='%s'", engine->numKeypoints(), camera_id.c_str()); return out; } @@ -670,4 +729,78 @@ namespace ANSCENTER { } } + // ==================================================================== + // TryLoadPersonDetector + // + // Look for person.onnx + person.names alongside the pose model. If + // both exist, instantiate an ANSONNXYOLO front-end and load it via + // LoadModelFromFolder (which creates the ORT session). On success + // m_personDetector is non-null and DetectObjects switches to the + // proper top-down pipeline. + // + // Missing files are NOT an error — the wrapper is designed to work + // standalone for single-person test inputs. We log a debug line so + // it's clear which path is active and return false. + // ==================================================================== + bool ANSONNXRTMPOSE::TryLoadPersonDetector(const std::string& modelFolder) { + const std::string personOnnx = CreateFilePath(modelFolder, "person.onnx"); + const std::string personNames = CreateFilePath(modelFolder, "person.names"); + if (!FileExist(personOnnx) || !FileExist(personNames)) { + ANS_DBG("ANSONNXRTMPOSE", + "TryLoadPersonDetector: person.onnx/person.names not in '%s' " + "-- standalone mode (whole-image pose)", modelFolder.c_str()); + return false; + } + if (m_personDetector) { + ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: already loaded, skipping"); + return true; + } + + try { + auto detector = std::make_unique(); + detector->SetLoadEngineOnCreation(this->_loadEngineOnCreation); + + ModelConfig personCfg{}; + personCfg.detectionType = DetectionType::DETECTION; + personCfg.modelType = ModelType::ONNXYOLO; + personCfg.precisionType = PrecisionType::FP32; + personCfg.inpHeight = 640; + personCfg.inpWidth = 640; + personCfg.detectionScoreThreshold = 0.5f; + personCfg.modelConfThreshold = 0.5f; + personCfg.modelMNSThreshold = 0.45f; + personCfg.numKPS = 0; + personCfg.gpuOptBatchSize = 1; + personCfg.gpuMaxBatchSize = 1; + + std::string personLabelMap; + const bool ok = detector->LoadModelFromFolder( + /*licenseKey*/ _licenseKey, + personCfg, + /*modelName*/ "person", + /*className*/ "person.names", + /*modelFolder*/ modelFolder, + personLabelMap); + if (!ok) { + _logger.LogWarn("ANSONNXRTMPOSE::TryLoadPersonDetector", + "Person detector LoadModelFromFolder FAILED -- falling back to whole-image pose", + __FILE__, __LINE__); + ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: detector load FAILED"); + return false; + } + + m_personDetector = std::move(detector); + ANS_DBG("ANSONNXRTMPOSE", + "TryLoadPersonDetector: detector loaded OK from '%s' (labels='%s')", + modelFolder.c_str(), personLabelMap.c_str()); + return true; + } + catch (const std::exception& e) { + _logger.LogWarn("ANSONNXRTMPOSE::TryLoadPersonDetector", e.what(), __FILE__, __LINE__); + ANS_DBG("ANSONNXRTMPOSE", "TryLoadPersonDetector: EXCEPTION '%s'", e.what()); + m_personDetector.reset(); + return false; + } + } + } // namespace ANSCENTER diff --git a/modules/ANSODEngine/ANSONNXRTMPOSE.h b/modules/ANSODEngine/ANSONNXRTMPOSE.h index ec3f1cb..f0e14b8 100644 --- a/modules/ANSODEngine/ANSONNXRTMPOSE.h +++ b/modules/ANSODEngine/ANSONNXRTMPOSE.h @@ -3,6 +3,7 @@ #pragma once #include "ANSEngineCommon.h" #include "ONNXEngine.h" +#include "ANSONNXYOLO.h" // internal person-detector front-end (top-down pipeline) namespace ANSCENTER { @@ -144,6 +145,21 @@ namespace ANSCENTER { // member here on another thread. std::shared_ptr m_ortEngine; + // ── Optional internal person detector (top-down pipeline) ─────── + // RTMPose is a top-down single-person estimator. If the model + // folder bundles person.onnx + person.names alongside the pose + // model, we auto-load an ANSONNXYOLO person detector and run pose + // ONCE PER DETECTED PERSON. When those files are missing the + // wrapper falls back to whole-image pose so single-person test + // inputs (selfies, fitness clips) still work. + std::unique_ptr m_personDetector; + + // Look for person.onnx + person.names in `modelFolder`. Loads + // m_personDetector and returns true on success; returns false + // (without logging an error) if either file is missing — caller + // should treat that as "no detector, fall back to whole-image". + bool TryLoadPersonDetector(const std::string& modelFolder); + bool InitOrtEngine(); bool InitOrtEngine(ANSCENTER::EngineType engineType); void WarmUpEngine(); diff --git a/modules/ANSODEngine/ANSRTRTMPOSE.cpp b/modules/ANSODEngine/ANSRTRTMPOSE.cpp index bee658e..5939481 100644 --- a/modules/ANSODEngine/ANSRTRTMPOSE.cpp +++ b/modules/ANSODEngine/ANSRTRTMPOSE.cpp @@ -46,6 +46,12 @@ namespace ANSCENTER { return false; } ANS_DBG("ANSRTRTMPOSE", "OptimizeModel: build OK -> '%s'", optimizedModelFolder.c_str()); + + // Also build the bundled person-detector engine if present. + // TryLoadPersonDetector triggers ANSRTYOLO::LoadModelFromFolder + // which performs the engine build (= optimization) and caches + // it to disk; the next Initialize/LoadModel will hit the cache. + (void)TryLoadPersonDetector(_modelFolder); return true; } catch (std::exception& e) { @@ -138,6 +144,11 @@ namespace ANSCENTER { ANS_DBG("ANSRTRTMPOSE", "LoadModel: engine ready %dx%d kps=%d sharedPool=%d", RTM_INPUT_W, RTM_INPUT_H, _modelConfig.numKPS, m_usingSharedPool ? 1 : 0); } + + // Top-down pipeline: try to load the bundled person detector. + // Falls back gracefully (whole-image pose) when the files are missing. + (void)TryLoadPersonDetector(_modelFolder); + _modelLoadValid = true; _isInitialized = true; return true; @@ -242,6 +253,9 @@ namespace ANSCENTER { ANS_DBG("ANSRTRTMPOSE", "LoadModelFromFolder: engine ready %dx%d kps=%d sharedPool=%d", RTM_INPUT_W, RTM_INPUT_H, _modelConfig.numKPS, m_usingSharedPool ? 1 : 0); } + + (void)TryLoadPersonDetector(_modelFolder); + _modelLoadValid = true; _isInitialized = true; return true; @@ -344,6 +358,9 @@ namespace ANSCENTER { ANS_DBG("ANSRTRTMPOSE", "Initialize: engine ready %dx%d kps=%d sharedPool=%d", RTM_INPUT_W, RTM_INPUT_H, _modelConfig.numKPS, m_usingSharedPool ? 1 : 0); } + + (void)TryLoadPersonDetector(_modelFolder); + _modelLoadValid = true; _isInitialized = true; return true; @@ -370,11 +387,56 @@ namespace ANSCENTER { return {}; } try { - // Top-down pose estimator: with no external bbox, treat the whole - // image as a single person crop. Higher-level orchestration that - // wants per-person pose should call ANSODBase::RunInference(input, - // bboxes, cam) instead — that overload crops each bbox and routes - // back through us per crop. + // Two pipelines: + // + // (a) Top-down (preferred) — m_personDetector loaded from + // the bundled person.onnx + person.names. Detect people + // first, then run RTMPose ONCE per person bbox. This is + // what RTMPose is designed for; works on multi-person + // scenes with small / distant subjects. + // + // (b) Standalone fallback — no detector files in the model + // folder. Treat the whole image as a single person crop. + // Only sensible for inputs that are already a person + // (selfies, single-subject fitness clips, unit-test + // crops). Higher-level callers can also use the + // ANSODBase::RunInference(input, vector, cam) + // overload which crops each bbox and routes per-crop + // through our Preprocess. + if (m_personDetector) { + std::vector persons = m_personDetector->RunInference(inputImgBGR, camera_id); + if (persons.empty()) { + ANS_DBG("ANSRTRTMPOSE", "RunInference: detector found 0 persons cam='%s'", + camera_id.c_str()); + return {}; + } + + std::vector results; + results.reserve(persons.size()); + for (const auto& person : persons) { + // person.onnx is a single-class model — accept anything + // the detector returned, but defend against degenerate + // boxes (zero/negative area, off-image). + if (person.box.width <= 0 || person.box.height <= 0) continue; + + auto poseResults = InferenceOnBox(inputImgBGR, person.box, camera_id); + for (auto& obj : poseResults) { + // Carry the detector's track-id and confidence so + // downstream tracking/visualization stays coherent + // across frames. RTMPose itself has no class / + // tracking concept of its own. + obj.trackId = person.trackId; + obj.confidence = person.confidence; + if (!person.className.empty()) obj.className = person.className; + results.push_back(std::move(obj)); + } + } + ANS_DBG("ANSRTRTMPOSE", "RunInference: %zu persons -> %zu poses cam='%s'", + persons.size(), results.size(), camera_id.c_str()); + return results; + } + + // Fallback — single-person whole-image pose const cv::Rect wholeImage(0, 0, inputImgBGR.cols, inputImgBGR.rows); return InferenceOnBox(inputImgBGR, wholeImage, camera_id); } @@ -394,6 +456,10 @@ namespace ANSCENTER { bool ANSRTRTMPOSE::Destroy() { try { + if (m_personDetector) { + m_personDetector->Destroy(); + m_personDetector.reset(); + } if (m_usingSharedPool) { EnginePoolManager::instance().release(m_poolKey); m_trtEngine.reset(); @@ -699,4 +765,83 @@ namespace ANSCENTER { cv::Size(RTM_INPUT_W, RTM_INPUT_H), cv::INTER_LINEAR); } + // ==================================================================== + // TryLoadPersonDetector + // + // Look for person.onnx + person.names alongside the pose model. If + // both exist, instantiate an ANSRTYOLO front-end and load it via + // LoadModelFromFolder (which builds / loads the TRT engine from + // cache). On success m_personDetector is non-null and RunInference + // switches to the proper top-down pipeline. + // + // Missing files are NOT an error — the wrapper is designed to work + // standalone for single-person test inputs. We log a debug line so + // it's clear which path is active and return false. + // ==================================================================== + bool ANSRTRTMPOSE::TryLoadPersonDetector(const std::string& modelFolder) { + const std::string personOnnx = CreateFilePath(modelFolder, "person.onnx"); + const std::string personNames = CreateFilePath(modelFolder, "person.names"); + if (!FileExist(personOnnx) || !FileExist(personNames)) { + ANS_DBG("ANSRTRTMPOSE", + "TryLoadPersonDetector: person.onnx/person.names not in '%s' " + "-- standalone mode (whole-image pose)", modelFolder.c_str()); + return false; + } + if (m_personDetector) { + ANS_DBG("ANSRTRTMPOSE", "TryLoadPersonDetector: already loaded, skipping"); + return true; + } + + try { + auto detector = std::make_unique(); + detector->SetLoadEngineOnCreation(this->_loadEngineOnCreation); + // SetMaxSlotsPerGpu is declared private in ANSRTYOLO (it's a + // private override of ANSODBase's public virtual). Call it via + // the base-class interface where access is public. + static_cast(detector.get())->SetMaxSlotsPerGpu(m_maxSlotsPerGpu); + + ModelConfig personCfg{}; + personCfg.detectionType = DetectionType::DETECTION; + personCfg.modelType = ModelType::RTYOLO; + personCfg.precisionType = PrecisionType::FP16; + personCfg.inpHeight = 640; + personCfg.inpWidth = 640; + personCfg.detectionScoreThreshold = 0.5f; + personCfg.modelConfThreshold = 0.5f; + personCfg.modelMNSThreshold = 0.45f; + personCfg.numKPS = 0; + personCfg.gpuOptBatchSize = 1; + personCfg.gpuMaxBatchSize = 1; + personCfg.gpuDeviceIndex = m_options.deviceIndex; + + std::string personLabelMap; + const bool ok = detector->LoadModelFromFolder( + /*licenseKey*/ _licenseKey, + personCfg, + /*modelName*/ "person", + /*className*/ "person.names", + /*modelFolder*/ modelFolder, + personLabelMap); + if (!ok) { + _logger.LogWarn("ANSRTRTMPOSE::TryLoadPersonDetector", + "Person detector LoadModelFromFolder FAILED -- falling back to whole-image pose", + __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "TryLoadPersonDetector: detector load FAILED"); + return false; + } + + m_personDetector = std::move(detector); + ANS_DBG("ANSRTRTMPOSE", + "TryLoadPersonDetector: detector loaded OK from '%s' (labels='%s')", + modelFolder.c_str(), personLabelMap.c_str()); + return true; + } + catch (const std::exception& e) { + _logger.LogWarn("ANSRTRTMPOSE::TryLoadPersonDetector", e.what(), __FILE__, __LINE__); + ANS_DBG("ANSRTRTMPOSE", "TryLoadPersonDetector: EXCEPTION '%s'", e.what()); + m_personDetector.reset(); + return false; + } + } + } // namespace ANSCENTER diff --git a/modules/ANSODEngine/ANSRTRTMPOSE.h b/modules/ANSODEngine/ANSRTRTMPOSE.h index f706d1f..d44b2df 100644 --- a/modules/ANSODEngine/ANSRTRTMPOSE.h +++ b/modules/ANSODEngine/ANSRTRTMPOSE.h @@ -5,6 +5,7 @@ #include "engine.h" #include "ANSGpuFrameRegistry.h" #include "engine/EnginePoolManager.h" +#include "ANSRTYOLO.h" // internal person-detector front-end (top-down pipeline) namespace ANSCENTER { @@ -100,6 +101,21 @@ namespace ANSCENTER { EnginePoolManager::PoolKey m_poolKey; bool m_usingSharedPool = false; + // ── Optional internal person detector (top-down pipeline) ─────── + // RTMPose is a top-down single-person estimator — feeding it whole + // scenes produces noise. If the model folder bundles person.onnx + + // person.names alongside the pose model, we auto-load a YOLO person + // detector here and run pose ONCE PER DETECTED PERSON. When those + // files are missing the wrapper falls back to whole-image pose so + // that single-person test inputs (selfies, fitness clips) still work. + std::unique_ptr m_personDetector; + + // Look for person.onnx + person.names in `modelFolder`. Loads + // m_personDetector and returns true on success; returns false + // (without logging an error) if either file is missing — caller + // should treat that as "no detector, fall back to whole-image". + bool TryLoadPersonDetector(const std::string& modelFolder); + ANSCENTER::Options m_options; // Fixed RTMPose model geometry (overridden from the engine's input dims diff --git a/tests/ANSODEngine-UnitTest/ANSODEngine-UnitTest.cpp b/tests/ANSODEngine-UnitTest/ANSODEngine-UnitTest.cpp index bd88aa1..b8ccc98 100644 --- a/tests/ANSODEngine-UnitTest/ANSODEngine-UnitTest.cpp +++ b/tests/ANSODEngine-UnitTest/ANSODEngine-UnitTest.cpp @@ -971,16 +971,19 @@ int VideoDetectorEngine(std::string modelFilePath, std::string videoFilePath, in printf("Inference Time = %lld ms\n", static_cast(elapsed.count())); if (!detectionResult.empty()) { - // COCO pose skeleton connections - const std::vector> POSE_SKELETON = { + // ── COCO 17-keypoint skeleton (Ultralytics YOLO Pose format) ── + // Indices: 0=nose, 1=Leye, 2=Reye, 3=Lear, 4=Rear, + // 5=Lshoulder, 6=Rshoulder, 7=Lelbow, 8=Relbow, + // 9=Lwrist, 10=Rwrist, 11=Lhip, 12=Rhip, + // 13=Lknee, 14=Rknee, 15=Lankle, 16=Rankle + const std::vector> COCO17_SKELETON = { {0,1}, {0,2}, {1,3}, {2,4}, // Face {3,5}, {4,6}, // Head-to-shoulder {5,7}, {7,9}, {6,8}, {8,10}, // Arms {5,6}, {5,11}, {6,12}, {11,12}, // Body {11,13}, {13,15}, {12,14}, {14,16} // Legs }; - // Colors for different limb groups - const std::vector LIMB_COLORS = { + const std::vector COCO17_COLORS = { {255,0,0}, {255,0,0}, {255,0,0}, {255,0,0}, // Face - blue {0,255,255}, {0,255,255}, // Head-shoulder - yellow {0,255,0}, {0,255,0}, {0,165,255}, {0,165,255}, // Arms - green/orange @@ -988,6 +991,68 @@ int VideoDetectorEngine(std::string modelFilePath, std::string videoFilePath, in {0,255,0}, {0,255,0}, {0,165,255}, {0,165,255} // Legs - green/orange }; + // ── Halpe 26-keypoint skeleton (MMPose body_2d_keypoint, used by RTMPose) ── + // Layout: 0..16 = COCO 17 (same as above); + // 17=Head, 18=Neck, 19=Hip(mid), + // 20=LBigToe, 21=RBigToe, + // 22=LSmallToe, 23=RSmallToe, + // 24=LHeel, 25=RHeel + const std::vector> HALPE26_SKELETON = { + {0,1}, {0,2}, {1,3}, {2,4}, // face + {17,18}, // head -> neck + {5,18}, {6,18}, // shoulders -> neck + {18,19}, // neck -> mid hip + {19,11}, {19,12}, // mid hip -> L/R hip + {5,7}, {7,9}, // left arm + {6,8}, {8,10}, // right arm + {11,13}, {13,15}, // left leg + {12,14}, {14,16}, // right leg + {15,20}, {15,22}, {15,24}, // left foot (big-toe / small-toe / heel) + {16,21}, {16,23}, {16,25} // right foot + }; + const std::vector HALPE26_COLORS = { + {255,0,0}, {255,0,0}, {255,0,0}, {255,0,0}, // face - blue + {0,255,255}, // head-neck - yellow + {0,255,255}, {0,255,255}, // shoulders-neck - yellow + {255,255,0}, // neck-midhip - cyan + {255,255,0}, {255,255,0}, // midhip-hips - cyan + {0,255,0}, {0,255,0}, // left arm - green + {0,165,255}, {0,165,255}, // right arm - orange + {0,255,0}, {0,255,0}, // left leg - green + {0,165,255}, {0,165,255}, // right leg - orange + {200,100,255}, {200,100,255}, {200,100,255}, // left foot - light purple + {255,100,100}, {255,100,100}, {255,100,100} // right foot - light pink + }; + + // ── OpenPose 18-keypoint skeleton (OpenVINO HumanPose, used by ANSPOSE) ── + // DIFFERENT index layout from COCO-17: + // 0=Nose, 1=Neck, 2=RShoulder, 3=RElbow, 4=RWrist, + // 5=LShoulder, 6=LElbow, 7=LWrist, + // 8=RHip, 9=RKnee, 10=RAnkle, + // 11=LHip, 12=LKnee, 13=LAnkle, + // 14=REye, 15=LEye, 16=REar, 17=LEar + // Note: ANSPOSE writes kps as stride-2 (x,y only — no per-kp score), + // so the drawer detects this layout by buffer size and skips + // the confidence threshold for it. + const std::vector> OPENPOSE18_SKELETON = { + {1,2}, {2,3}, {3,4}, // right arm (neck->shoulder->elbow->wrist) + {1,5}, {5,6}, {6,7}, // left arm + {1,8}, {8,9}, {9,10}, // right leg (neck->hip->knee->ankle) + {1,11}, {11,12}, {12,13}, // left leg + {1,0}, // neck->nose + {0,14}, {14,16}, // right eye->ear + {0,15}, {15,17} // left eye->ear + }; + const std::vector OPENPOSE18_COLORS = { + {0,165,255}, {0,165,255}, {0,165,255}, // right arm - orange + {0,255,0}, {0,255,0}, {0,255,0}, // left arm - green + {0,165,255}, {0,165,255}, {0,165,255}, // right leg - orange + {0,255,0}, {0,255,0}, {0,255,0}, // left leg - green + {255,255,0}, // neck-nose - cyan + {255,0,0}, {255,0,0}, // right face - blue + {255,0,0}, {255,0,0} // left face - blue + }; + pt.clear(); std::stringstream ss; ss.clear(); @@ -1103,29 +1168,88 @@ int VideoDetectorEngine(std::string modelFilePath, std::string videoFilePath, in } } - // Draw keypoints if available (format: x1;y1;score1;x2;y2;score2;... stride=3) + // Draw keypoints if available. Three on-the-wire formats are + // auto-detected from the buffer length so the same drawer + // handles all pose models without changing the engines: + // + // 51 floats → 17 kps stride-3 → YOLO Pose (COCO-17) + // 78 floats → 26 kps stride-3 → RTMPose (Halpe-26) + // 36 floats → 18 kps stride-2 → OpenPose/ANSPOSE + // + // For stride-3 formats kps[k*3+2] is a per-keypoint confidence + // and we threshold on it. ANSPOSE doesn't emit confidences + // (kps are stride-2), so we use kp.x>0 && kp.y>0 as the + // visibility proxy (ANSPOSE writes 0 / negatives for missing + // keypoints — see ANSPOSE.cpp:292). if (!kps_str.empty()) { std::vector kps = ANSCENTER::ANSUtilityHelper::StringToKeypoints(kps_str); - int numKeypoints = (int)kps.size() / 3; // stride of 3: x, y, confidence + const size_t n = kps.size(); - // Collect keypoint positions and confidence + const std::vector>* skeleton = nullptr; + const std::vector* colors = nullptr; + int stride = 3; + int numKeypoints = 0; + bool hasPerKpScore = true; + + if (n == 51) { // YOLO Pose: 17 kps × 3 + stride = 3; + numKeypoints = 17; + skeleton = &COCO17_SKELETON; + colors = &COCO17_COLORS; + hasPerKpScore = true; + } + else if (n == 78) { // RTMPose: 26 kps × 3 + stride = 3; + numKeypoints = 26; + skeleton = &HALPE26_SKELETON; + colors = &HALPE26_COLORS; + hasPerKpScore = true; + } + else if (n == 36) { // OpenPose/ANSPOSE: 18 kps × 2 + stride = 2; + numKeypoints = 18; + skeleton = &OPENPOSE18_SKELETON; + colors = &OPENPOSE18_COLORS; + hasPerKpScore = false; + } + else if (n % 3 == 0) { // Unknown stride-3 — draw dots only + stride = 3; + numKeypoints = static_cast(n) / 3; + hasPerKpScore = true; + } + else if (n % 2 == 0) { // Unknown stride-2 — draw dots only + stride = 2; + numKeypoints = static_cast(n) / 2; + hasPerKpScore = false; + } + + // Collect keypoint positions and (optional) confidence. + // For stride-2 we treat any kp with x>0 && y>0 as visible. std::vector keypointPts(numKeypoints); - std::vector keypointConf(numKeypoints, 0.f); + std::vector keypointConf(numKeypoints, 1.f); for (int k = 0; k < numKeypoints; k++) { - keypointPts[k] = cv::Point2f(kps[k * 3], kps[k * 3 + 1]); - keypointConf[k] = kps[k * 3 + 2]; + const float kx = kps[k * stride]; + const float ky = kps[k * stride + 1]; + keypointPts[k] = cv::Point2f(kx, ky); + if (hasPerKpScore) { + keypointConf[k] = kps[k * stride + 2]; + } else { + keypointConf[k] = (kx > 0.f && ky > 0.f) ? 1.f : 0.f; + } } const float kpConfThreshold = 0.3f; - // Draw skeleton lines - for (size_t s = 0; s < POSE_SKELETON.size(); s++) { - int idx1 = POSE_SKELETON[s].first; - int idx2 = POSE_SKELETON[s].second; - if (idx1 < numKeypoints && idx2 < numKeypoints && - keypointConf[idx1] > kpConfThreshold && keypointConf[idx2] > kpConfThreshold) { - cv::Scalar color = (s < LIMB_COLORS.size()) ? LIMB_COLORS[s] : cv::Scalar(200, 200, 200); - cv::line(frame, keypointPts[idx1], keypointPts[idx2], color, 2, cv::LINE_AA); + // Draw skeleton lines (skipped for unknown layouts). + if (skeleton) { + for (size_t s = 0; s < skeleton->size(); s++) { + int idx1 = (*skeleton)[s].first; + int idx2 = (*skeleton)[s].second; + if (idx1 < numKeypoints && idx2 < numKeypoints && + keypointConf[idx1] > kpConfThreshold && keypointConf[idx2] > kpConfThreshold) { + cv::Scalar color = (s < colors->size()) ? (*colors)[s] : cv::Scalar(200, 200, 200); + cv::line(frame, keypointPts[idx1], keypointPts[idx2], color, 2, cv::LINE_AA); + } } } @@ -1488,7 +1612,6 @@ int YOLO26POSEYolo11Test() { int modelType = 31; // ONNX YOLO (30) RT YOLO (31) std::string onnxclmodelFilePath = "E:\\Programs\\DemoAssets\\ANSAIModels\\PoseEstimationModels\\ANSPOSEN.zip"; std::string videoFile = "E:\\Programs\\DemoAssets\\Videos\\train.mp4"; - VideoDetectorEngine(onnxclmodelFilePath, videoFile, modelType);// 22 is RT, 21 is onnx return 0; } @@ -1872,14 +1995,23 @@ int FallDetectorTest() { VideoDetectorEngine(modelFilePath, videoFile, modelType); return 0; } + +int RTMPOSETest() { + int modelType = 33; // ONNX YOLO (30) RT YOLO (31) + std::string onnxclmodelFilePath = "E:\\Programs\\DemoAssets\\ANSAIModels\\PoseEstimationModels\\ANSRTMPOSE.zip"; + std::string videoFile = "E:\\Programs\\DemoAssets\\Videos\\train.mp4"; + VideoDetectorEngine(onnxclmodelFilePath, videoFile, modelType);// 22 is RT, 21 is onnx + return 0; +} int main() { - FallDetectorTest(); + //FallDetectorTest(); //ALPROD_Test(); //FireNSmokeTest(); //ElasticGPUTest(); + RTMPOSETest(); //YOLO26SEGYolo11Test(); //YOLO26POSEYolo11Test(); //YOLO26CLYolo11Test();